From 38f75d880e04865c8bd620aabda574fc84dccef3 Mon Sep 17 00:00:00 2001 From: Bas des Tombe Date: Wed, 15 Jul 2026 12:19:30 +0200 Subject: [PATCH] Add DAWACO well access log readers Add get_daw_sensorchange for DrukmetW sensor changes and get_daw_accesstowell for combined well access events across sensor changes, hand measurements, Refpunt adjustments, and water-quality samples. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dawacotools/__init__.py | 4 + dawacotools/io.py | 268 ++++++++++++++++++++++++++++++++++++++++ tests/mock_dawaco.py | 92 ++++++++++++-- tests/test_io.py | 142 +++++++++++++++++++++ 4 files changed, 498 insertions(+), 8 deletions(-) diff --git a/dawacotools/__init__.py b/dawacotools/__init__.py index be40a78..0d14706 100644 --- a/dawacotools/__init__.py +++ b/dawacotools/__init__.py @@ -7,12 +7,14 @@ create_dawaco_engine, df2gdf, get_connection_string, + get_daw_accesstowell, get_daw_boring, get_daw_coords_from_mpcode, get_daw_filters, get_daw_meteo_from_loc, get_daw_mon_dates, get_daw_mps, + get_daw_sensorchange, get_daw_soort_mp, get_daw_triwaco, get_daw_ts_meteo, @@ -32,12 +34,14 @@ "df2gdf", "get_cluster_mps", "get_connection_string", + "get_daw_accesstowell", "get_daw_boring", "get_daw_coords_from_mpcode", "get_daw_filters", "get_daw_meteo_from_loc", "get_daw_mon_dates", "get_daw_mps", + "get_daw_sensorchange", "get_daw_soort_mp", "get_daw_triwaco", "get_daw_ts_meteo", diff --git a/dawacotools/io.py b/dawacotools/io.py index 2109449..79a079f 100644 --- a/dawacotools/io.py +++ b/dawacotools/io.py @@ -365,6 +365,36 @@ def fuzzy_match_mpcode( return query +def _filter_selection_where_clause( + table_alias: str, + mpcode=None, + filternr=None, + *, + partial_match_mpcode=True, +) -> tuple[str, dict[str, object]]: + conditions = [] + params: dict[str, object] = {} + + if mpcode is not None: + column_name = f"{table_alias}.MpCode" + mpcodes = _matching_mpcodes(mpcode, partial_match_mpcode=partial_match_mpcode) + mpcode_clause, mpcode_params = _sql_in_clause(column_name, mpcodes, "mpcode") + conditions.append(mpcode_clause) + params.update(mpcode_params) + + if filternr is not None: + filternr_clause, filternr_params = _sql_in_clause( + f"{table_alias}.Filtnr", + _normalise_filternrs(filternr), + "filternr", + ) + conditions.append(filternr_clause) + params.update(filternr_params) + + where_clause = "" if len(conditions) == 0 else "WHERE " + " AND ".join(conditions) + return where_clause, params + + def get_daw_filters( mpcode=None, filternr=None, @@ -773,6 +803,244 @@ def get_daw_ts_stijghgt(mpcode=None, filternr=None): return identify_data_gaps(out) +def get_daw_sensorchange(mpcode=None, filternr=None, typechange="inout", *, partial_match_mpcode=True): + """ + Return sensor change dates for monitoring point filters. + + Parameters + ---------- + mpcode : str or iterable of str, optional + Monitoring point code selection. By default, values are matched as + substrings, matching the fuzzy behavior of ``get_daw_filters``. + filternr : int or iterable of int, optional + Filter number selection. + typechange : {"inout", "in", "out"}, default "inout" + Select all sensor changes, only sensor placements, or only removals. + partial_match_mpcode : bool, default True + Whether to match ``mpcode`` values by substring. + + Returns + ------- + pandas.DataFrame + Sensor changes with columns ``Datum``, ``MpCode``, ``Filtnr``, and + ``Type_Wijz``. ``Datum`` combines the DAWACO date and time columns. + Rows are sorted by timestamp with a consecutive integer index. + """ + typechange_options = {"inout": None, "in": "I", "out": "O"} + try: + typechange_value = typechange_options[typechange.lower()] + except (AttributeError, KeyError): + msg = "typechange must be one of 'inout', 'in', or 'out'" + raise ValueError(msg) from None + + where_clause, params = _filter_selection_where_clause( + "DrukmetW", + mpcode=mpcode, + filternr=filternr, + partial_match_mpcode=partial_match_mpcode, + ) + if typechange_value is not None: + prefix = "WHERE " if not where_clause else " AND " + where_clause += prefix + "DrukmetW.Type_Wijz = :typechange" + params["typechange"] = typechange_value + + query = ( + _sql( # noqa: S608 + """ + SELECT Datum, Tijd, MpCode, Filtnr, Type_Wijz + FROM {DrukmetW} as DrukmetW + """, + DrukmetW=_table("DrukmetW"), + ) + + where_clause + + "\nORDER BY Datum, Tijd" + ) + + sensor_changes = _read_sql_query(query, params=params, dtype={"Filtnr": int}) + sensor_changes["Datum"] = pd.to_datetime( + sensor_changes["Datum"].astype(str) + " " + sensor_changes.pop("Tijd").astype(str), + errors="coerce", + ) + return sensor_changes.sort_values("Datum", kind="stable").reset_index(drop=True) + + +def _datetime_from_date_time(dataframe: pd.DataFrame, date_column: str, time_column: str) -> pd.Series: + return pd.to_datetime( + dataframe[date_column].astype(str) + " " + dataframe.pop(time_column).fillna("").astype(str), + errors="coerce", + ) + + +def _get_daw_joined_filter_access( + table_name: str, + source_label: str, + mpcode=None, + filternr=None, + *, + partial_match_mpcode=True, + table_alias="AccessLog", + date_column="Datum", + time_column="Tijd", + extra_condition: str | None = None, + extra_params: dict[str, object] | None = None, +) -> pd.DataFrame: + filters_alias = "Filters" + where_clause, params = _filter_selection_where_clause( + filters_alias, + mpcode=mpcode, + filternr=filternr, + partial_match_mpcode=partial_match_mpcode, + ) + if extra_condition is not None: + prefix = "WHERE " if not where_clause else " AND " + where_clause += prefix + extra_condition + if extra_params is not None: + params.update(extra_params) + + query = ( + _sql( # noqa: S608 + """ + SELECT {AccessLog}.{date_column} AS Datum, {AccessLog}.{time_column} AS Tijd, Filters.MpCode, Filters.Filtnr + FROM {access_log} as {AccessLog} + INNER JOIN {filters} as Filters on Filters.RECNUM = {AccessLog}.Filtrec + """, + AccessLog=table_alias, + date_column=date_column, + time_column=time_column, + access_log=_table(table_name), + filters=_table("Filters"), + ) + + where_clause + + "\nORDER BY " + + table_alias + + "." + + date_column + + ", " + + table_alias + + "." + + time_column + ) + + access_log = _read_sql_query(query, params=params, dtype={"Filtnr": int}) + access_log["Datum"] = _datetime_from_date_time(access_log, "Datum", "Tijd") + access_log["Type"] = source_label + return access_log + + +def _get_daw_refpunt_adjustments(mpcode=None, filternr=None, *, partial_match_mpcode=True) -> pd.DataFrame: + where_clause, params = _filter_selection_where_clause( + "Filters", + mpcode=mpcode, + filternr=filternr, + partial_match_mpcode=partial_match_mpcode, + ) + prefix = "WHERE " if not where_clause else " AND " + where_clause += prefix + "Refpunt.Type = :refpunt_type" + params["refpunt_type"] = "A" + + query = ( + _sql( # noqa: S608 + """ + SELECT Refpunt.Datum AS Datum, Refpunt.Tijd AS Tijd, Filters.MpCode, Filters.Filtnr + FROM {refpunt} as Refpunt + INNER JOIN {filters} as Filters on Filters.RECNUM = Refpunt.Filtrec + """, + refpunt=_table("Refpunt"), + filters=_table("Filters"), + ) + + where_clause + + "\nORDER BY Datum, Tijd" + ) + refpunt = _read_sql_query(query, params=params, dtype={"Filtnr": int}) + refpunt["Datum"] = _datetime_from_date_time(refpunt, "Datum", "Tijd") + refpunt["Type"] = "refpunt_adjustment" + return refpunt + + +def _get_daw_water_quality_samples(mpcode=None, filternr=None, *, partial_match_mpcode=True) -> pd.DataFrame: + where_clause, params = _filter_selection_where_clause( + "Filters", + mpcode=mpcode, + filternr=filternr, + partial_match_mpcode=partial_match_mpcode, + ) + + query = ( + _sql( # noqa: S608 + """ + SELECT GwkMon.datum AS Datum, Filters.MpCode, Filters.Filtnr + FROM {gwkmon} as GwkMon + INNER JOIN {filters} as Filters on Filters.RECNUM = GwkMon.Filtrec + """, + gwkmon=_table("gwkmon"), + filters=_table("Filters"), + ) + + where_clause + + "\nORDER BY GwkMon.datum" + ) + + water_quality_samples = _read_sql_query(query, params=params, dtype={"Filtnr": int}) + water_quality_samples["Datum"] = pd.to_datetime(water_quality_samples["Datum"], errors="coerce") + water_quality_samples["Type"] = "water_quality_sample" + return water_quality_samples + + +def get_daw_accesstowell(mpcode=None, filternr=None, *, partial_match_mpcode=True): + """ + Return logged dates when a monitoring well was accessed. + + The access log combines sensor changes, validated hand measurements, hand + measurements, reference-height adjustments, and water-quality samples. + """ + sensor_changes = get_daw_sensorchange( + mpcode=mpcode, + filternr=filternr, + partial_match_mpcode=partial_match_mpcode, + ).rename(columns={"Type_Wijz": "Type"}) + sensor_changes["Type"] = sensor_changes["Type"].map({"I": "sensorchange_in", "O": "sensorchange_out"}) + + access_logs = [ + sensor_changes, + _get_daw_joined_filter_access( + "StygCont", + "validated_hand_measurement", + mpcode=mpcode, + filternr=filternr, + partial_match_mpcode=partial_match_mpcode, + date_column="Cont_Dat", + time_column="Cont_Tijd", + ), + _get_daw_joined_filter_access( + "Stijghgt", + "hand_measurement", + mpcode=mpcode, + filternr=filternr, + partial_match_mpcode=partial_match_mpcode, + date_column="datum", + time_column="tijd", + extra_condition="AccessLog.Bron = :bron", + extra_params={"bron": "V"}, + ), + _get_daw_water_quality_samples( + mpcode=mpcode, + filternr=filternr, + partial_match_mpcode=partial_match_mpcode, + ), + ] + access_logs.append( + _get_daw_refpunt_adjustments( + mpcode=mpcode, + filternr=filternr, + partial_match_mpcode=partial_match_mpcode, + ) + ) + + access_to_well = pd.concat(access_logs, ignore_index=True) + access_to_well = access_to_well.loc[:, ["Datum", "MpCode", "Filtnr", "Type"]] + access_to_well["Filtnr"] = access_to_well["Filtnr"].astype("Int64") + return access_to_well.sort_values("Datum", kind="stable").reset_index(drop=True) + + def get_daw_ts_temp(mpcode=None, filternr=None): """Return temperature measurements for a monitoring point filter.""" if mpcode is None or filternr is None: diff --git a/tests/mock_dawaco.py b/tests/mock_dawaco.py index 1c593b3..6b536be 100644 --- a/tests/mock_dawaco.py +++ b/tests/mock_dawaco.py @@ -24,6 +24,9 @@ def build_mock_dawaco_database(database_path: Path) -> Engine: _write_monitoring_points(connection) _write_filters(connection) _write_groundwater_levels(connection) + _write_validated_hand_measurements(connection) + _write_sensor_changes(connection) + _write_refpunt_adjustments(connection) _write_monitoring_dates(connection) _write_meteo(connection) _write_boring(connection) @@ -117,22 +120,95 @@ def _write_filters(connection: Connection) -> None: def _write_groundwater_levels(connection: Connection) -> None: pd.DataFrame([ - {"filtrec": 101, "datum": "2020-01-01", "tijd": "00:00", "meting_nap": 1.00, "Temp": 8.0}, - {"filtrec": 101, "datum": "2020-01-02", "tijd": "00:00", "meting_nap": 1.10, "Temp": 0.0}, - {"filtrec": 101, "datum": "2020-01-05", "tijd": "00:00", "meting_nap": -999.0, "Temp": -99.0}, - {"filtrec": 101, "datum": "2020-01-06", "tijd": "00:00", "meting_nap": 1.40, "Temp": 9.0}, - {"filtrec": 102, "datum": "2020-01-01", "tijd": "00:00", "meting_nap": 0.50, "Temp": 7.5}, - {"filtrec": 103, "datum": "2020-01-01", "tijd": "00:00", "meting_nap": 0.75, "Temp": 7.0}, - {"filtrec": 103, "datum": "2020-01-03", "tijd": "00:00", "meting_nap": 0.95, "Temp": 7.2}, - {"filtrec": 103, "datum": "2020-01-06", "tijd": "00:00", "meting_nap": 1.35, "Temp": 7.4}, + {"filtrec": 101, "datum": "2020-01-01", "tijd": "00:00", "meting_nap": 1.00, "Temp": 8.0, "Bron": "A"}, + {"filtrec": 101, "datum": "2020-01-02", "tijd": "00:00", "meting_nap": 1.10, "Temp": 0.0, "Bron": "V"}, + {"filtrec": 101, "datum": "2020-01-05", "tijd": "00:00", "meting_nap": -999.0, "Temp": -99.0, "Bron": "A"}, + {"filtrec": 101, "datum": "2020-01-06", "tijd": "00:00", "meting_nap": 1.40, "Temp": 9.0, "Bron": "A"}, + {"filtrec": 102, "datum": "2020-01-01", "tijd": "00:00", "meting_nap": 0.50, "Temp": 7.5, "Bron": "A"}, + {"filtrec": 103, "datum": "2020-01-01", "tijd": "00:00", "meting_nap": 0.75, "Temp": 7.0, "Bron": "A"}, + {"filtrec": 103, "datum": "2020-01-03", "tijd": "00:00", "meting_nap": 0.95, "Temp": 7.2, "Bron": "V"}, + {"filtrec": 103, "datum": "2020-01-06", "tijd": "00:00", "meting_nap": 1.35, "Temp": 7.4, "Bron": "A"}, ]).to_sql("Stijghgt", connection, index=False, if_exists="replace") +def _write_validated_hand_measurements(connection: Connection) -> None: + pd.DataFrame([ + {"Filtrec": 101, "Cont_Dat": "2020-01-02", "Cont_Tijd": "07:30"}, + {"Filtrec": 103, "Cont_Dat": "2020-01-04", "Cont_Tijd": "12:00"}, + {"Filtrec": 104, "Cont_Dat": "2020-01-05", "Cont_Tijd": "13:00"}, + ]).to_sql("StygCont", connection, index=False, if_exists="replace") + + +def _write_sensor_changes(connection: Connection) -> None: + pd.DataFrame([ + { + "Recnum": 401, + "Dm_Rec": 101, + "Datum": "2020-01-03", + "Tijd": "08:00", + "Type_Wijz": "O", + "MpCode": "MOCK001", + "Filtnr": 1, + "Opmerking": "Synthetic sensor removed", + }, + { + "Recnum": 402, + "Dm_Rec": 101, + "Datum": "2020-01-01", + "Tijd": "09:00", + "Type_Wijz": "I", + "MpCode": "MOCK001", + "Filtnr": 1, + "Opmerking": "Synthetic sensor placed", + }, + { + "Recnum": 403, + "Dm_Rec": 102, + "Datum": "2020-01-01", + "Tijd": "10:00", + "Type_Wijz": "I", + "MpCode": "MOCK001", + "Filtnr": 2, + "Opmerking": "Synthetic sensor placed", + }, + { + "Recnum": 404, + "Dm_Rec": 103, + "Datum": "2020-01-04", + "Tijd": "11:00", + "Type_Wijz": "O", + "MpCode": "MOCK002", + "Filtnr": 1, + "Opmerking": "Synthetic sensor removed", + }, + { + "Recnum": 405, + "Dm_Rec": 104, + "Datum": "2020-01-05", + "Tijd": "12:00", + "Type_Wijz": "I", + "MpCode": "MOCK010", + "Filtnr": 1, + "Opmerking": "Synthetic sensor placed", + }, + ]).to_sql("DrukmetW", connection, index=False, if_exists="replace") + + +def _write_refpunt_adjustments(connection: Connection) -> None: + pd.DataFrame([ + {"Filtrec": 101, "Datum": "2020-01-02", "Tijd": "06:00", "Type": "A"}, + {"Filtrec": 102, "Datum": "2020-01-02", "Tijd": "06:00", "Type": "A"}, + {"Filtrec": 103, "Datum": "2020-01-04", "Tijd": "10:30", "Type": "A"}, + {"Filtrec": 104, "Datum": "2020-01-05", "Tijd": "14:00", "Type": "B"}, + ]).to_sql("Refpunt", connection, index=False, if_exists="replace") + + def _write_monitoring_dates(connection: Connection) -> None: pd.DataFrame([ {"filtrec": 101, "datum": "2021-01-01"}, {"filtrec": 101, "datum": "2021-01-15"}, {"filtrec": 102, "datum": "2021-02-01"}, + {"filtrec": 103, "datum": "2021-03-01"}, ]).to_sql("gwkmon", connection, index=False, if_exists="replace") diff --git a/tests/test_io.py b/tests/test_io.py index d3332bf..2a3e64b 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -98,6 +98,148 @@ def test_get_daw_filters_can_return_hydropandas_metadata_shape(): assert not filters.iloc[0]["vervallen"] +def test_get_daw_sensorchange_returns_sorted_dataframe_with_integer_index(): + sensor_changes = dt.get_daw_sensorchange() + + assert list(sensor_changes.columns) == ["Datum", "MpCode", "Filtnr", "Type_Wijz"] + assert isinstance(sensor_changes.index, pd.RangeIndex) + assert list(sensor_changes.index) == [0, 1, 2, 3, 4] + assert list(sensor_changes["Datum"]) == [ + pd.Timestamp("2020-01-01 09:00"), + pd.Timestamp("2020-01-01 10:00"), + pd.Timestamp("2020-01-03 08:00"), + pd.Timestamp("2020-01-04 11:00"), + pd.Timestamp("2020-01-05 12:00"), + ] + assert list(sensor_changes[["MpCode", "Filtnr", "Type_Wijz"]].itertuples(index=False, name=None)) == [ + ("MOCK001", 1, "I"), + ("MOCK001", 2, "I"), + ("MOCK001", 1, "O"), + ("MOCK002", 1, "O"), + ("MOCK010", 1, "I"), + ] + + +def test_get_daw_sensorchange_supports_fuzzy_matching_filter_and_type_selection(): + partial = dt.get_daw_sensorchange(mpcode="MOCK00") + exact_empty = dt.get_daw_sensorchange(mpcode="MOCK00", partial_match_mpcode=False) + exact = dt.get_daw_sensorchange(mpcode="MOCK001", partial_match_mpcode=False) + exact_list = dt.get_daw_sensorchange(mpcode=["MOCK001", "MOCK002"], partial_match_mpcode=False) + filter_one_as_integer = dt.get_daw_sensorchange(mpcode="MOCK001", filternr=1) + filter_one = dt.get_daw_sensorchange(mpcode="MOCK001", filternr="1") + filter_list = dt.get_daw_sensorchange(mpcode="MOCK001", filternr=[1.0, 2.0]) + sensor_in = dt.get_daw_sensorchange(mpcode="MOCK001", typechange="in") + sensor_out = dt.get_daw_sensorchange(mpcode="MOCK001", typechange="out") + + expected_partial = [ + (pd.Timestamp("2020-01-01 09:00"), "MOCK001", 1, "I"), + (pd.Timestamp("2020-01-01 10:00"), "MOCK001", 2, "I"), + (pd.Timestamp("2020-01-03 08:00"), "MOCK001", 1, "O"), + (pd.Timestamp("2020-01-04 11:00"), "MOCK002", 1, "O"), + ] + assert list(partial.itertuples(index=False, name=None)) == expected_partial + assert exact_empty.empty + assert list(exact["MpCode"]) == ["MOCK001", "MOCK001", "MOCK001"] + assert list(exact_list.itertuples(index=False, name=None)) == expected_partial + assert list(filter_one_as_integer[["Filtnr", "Type_Wijz"]].itertuples(index=False, name=None)) == [ + (1, "I"), + (1, "O"), + ] + assert list(filter_one[["Filtnr", "Type_Wijz"]].itertuples(index=False, name=None)) == [(1, "I"), (1, "O")] + assert list(filter_list["Filtnr"]) == [1, 2, 1] + assert list(sensor_in["Type_Wijz"]) == ["I", "I"] + assert list(sensor_out["Type_Wijz"]) == ["O"] + + +def test_get_daw_sensorchange_validates_public_filters_and_binds_values_safely(): + injected = dt.get_daw_sensorchange(mpcode="MOCK001' OR '1'='1", partial_match_mpcode=False) + wildcard_percent = dt.get_daw_sensorchange(mpcode="MOCK%") + wildcard_underscore = dt.get_daw_sensorchange(mpcode="MOCK_") + lowercase_partial = dt.get_daw_sensorchange(mpcode="mock") + + assert injected.empty + assert wildcard_percent.empty + assert wildcard_underscore.empty + assert lowercase_partial.empty + with pytest.raises(ValueError, match="filternr must be a non-negative integer-like value"): + dt.get_daw_sensorchange(mpcode="MOCK001", filternr=-1) + with pytest.raises(ValueError, match="typechange must be one of"): + dt.get_daw_sensorchange(mpcode="MOCK001", typechange="unknown") + + +def test_get_daw_accesstowell_combines_access_sources_sorted_by_timestamp(): + access_log = dt.get_daw_accesstowell() + + assert list(access_log.columns) == ["Datum", "MpCode", "Filtnr", "Type"] + assert isinstance(access_log.index, pd.RangeIndex) + assert list(access_log.itertuples(index=False, name=None)) == [ + (pd.Timestamp("2020-01-01 09:00"), "MOCK001", 1, "sensorchange_in"), + (pd.Timestamp("2020-01-01 10:00"), "MOCK001", 2, "sensorchange_in"), + (pd.Timestamp("2020-01-02 00:00"), "MOCK001", 1, "hand_measurement"), + (pd.Timestamp("2020-01-02 06:00"), "MOCK001", 1, "refpunt_adjustment"), + (pd.Timestamp("2020-01-02 06:00"), "MOCK001", 2, "refpunt_adjustment"), + (pd.Timestamp("2020-01-02 07:30"), "MOCK001", 1, "validated_hand_measurement"), + (pd.Timestamp("2020-01-03 00:00"), "MOCK002", 1, "hand_measurement"), + (pd.Timestamp("2020-01-03 08:00"), "MOCK001", 1, "sensorchange_out"), + (pd.Timestamp("2020-01-04 10:30"), "MOCK002", 1, "refpunt_adjustment"), + (pd.Timestamp("2020-01-04 11:00"), "MOCK002", 1, "sensorchange_out"), + (pd.Timestamp("2020-01-04 12:00"), "MOCK002", 1, "validated_hand_measurement"), + (pd.Timestamp("2020-01-05 12:00"), "MOCK010", 1, "sensorchange_in"), + (pd.Timestamp("2020-01-05 13:00"), "MOCK010", 1, "validated_hand_measurement"), + (pd.Timestamp("2021-01-01 00:00"), "MOCK001", 1, "water_quality_sample"), + (pd.Timestamp("2021-01-15 00:00"), "MOCK001", 1, "water_quality_sample"), + (pd.Timestamp("2021-02-01 00:00"), "MOCK001", 2, "water_quality_sample"), + (pd.Timestamp("2021-03-01 00:00"), "MOCK002", 1, "water_quality_sample"), + ] + + +def test_get_daw_accesstowell_supports_fuzzy_mpcode_and_filter_selection(): + partial = dt.get_daw_accesstowell(mpcode="MOCK00") + lowercase_partial = dt.get_daw_accesstowell(mpcode="mock") + exact_empty = dt.get_daw_accesstowell(mpcode="MOCK00", partial_match_mpcode=False) + exact = dt.get_daw_accesstowell(mpcode="MOCK001", partial_match_mpcode=False) + filter_one = dt.get_daw_accesstowell(mpcode="MOCK001", filternr=1) + filter_one_as_string = dt.get_daw_accesstowell(mpcode="MOCK001", filternr="1") + filter_list = dt.get_daw_accesstowell(mpcode="MOCK001", filternr=[1.0, 2.0]) + + expected_partial = [ + (pd.Timestamp("2020-01-01 09:00"), "MOCK001", 1, "sensorchange_in"), + (pd.Timestamp("2020-01-01 10:00"), "MOCK001", 2, "sensorchange_in"), + (pd.Timestamp("2020-01-02 00:00"), "MOCK001", 1, "hand_measurement"), + (pd.Timestamp("2020-01-02 06:00"), "MOCK001", 1, "refpunt_adjustment"), + (pd.Timestamp("2020-01-02 06:00"), "MOCK001", 2, "refpunt_adjustment"), + (pd.Timestamp("2020-01-02 07:30"), "MOCK001", 1, "validated_hand_measurement"), + (pd.Timestamp("2020-01-03 00:00"), "MOCK002", 1, "hand_measurement"), + (pd.Timestamp("2020-01-03 08:00"), "MOCK001", 1, "sensorchange_out"), + (pd.Timestamp("2020-01-04 10:30"), "MOCK002", 1, "refpunt_adjustment"), + (pd.Timestamp("2020-01-04 11:00"), "MOCK002", 1, "sensorchange_out"), + (pd.Timestamp("2020-01-04 12:00"), "MOCK002", 1, "validated_hand_measurement"), + (pd.Timestamp("2021-01-01 00:00"), "MOCK001", 1, "water_quality_sample"), + (pd.Timestamp("2021-01-15 00:00"), "MOCK001", 1, "water_quality_sample"), + (pd.Timestamp("2021-02-01 00:00"), "MOCK001", 2, "water_quality_sample"), + (pd.Timestamp("2021-03-01 00:00"), "MOCK002", 1, "water_quality_sample"), + ] + expected_filter_one = [ + (pd.Timestamp("2020-01-01 09:00"), "MOCK001", 1, "sensorchange_in"), + (pd.Timestamp("2020-01-02 00:00"), "MOCK001", 1, "hand_measurement"), + (pd.Timestamp("2020-01-02 06:00"), "MOCK001", 1, "refpunt_adjustment"), + (pd.Timestamp("2020-01-02 07:30"), "MOCK001", 1, "validated_hand_measurement"), + (pd.Timestamp("2020-01-03 08:00"), "MOCK001", 1, "sensorchange_out"), + (pd.Timestamp("2021-01-01 00:00"), "MOCK001", 1, "water_quality_sample"), + (pd.Timestamp("2021-01-15 00:00"), "MOCK001", 1, "water_quality_sample"), + ] + + assert list(partial.itertuples(index=False, name=None)) == expected_partial + assert lowercase_partial.empty + assert exact_empty.empty + assert list(exact.itertuples(index=False, name=None)) == [row for row in expected_partial if row[1] == "MOCK001"] + assert list(filter_one.itertuples(index=False, name=None)) == expected_filter_one + assert list(filter_one_as_string.itertuples(index=False, name=None)) == expected_filter_one + assert list(filter_list.itertuples(index=False, name=None)) == [ + row for row in expected_partial if row[1] == "MOCK001" + ] + + def test_get_daw_mon_dates_returns_unique_sorted_dates(): dates = dt.get_daw_mon_dates(mpcode="MOCK001", filternr=1)