feat: handle floored downsampling posts

Signed-off-by: Mohamed Belhsan Hmida <mohamedbelhsanhmida@gmail.com>

flexmeasures/api/common/schemas/sensor_data.py

@@ -369,7 +369,8 @@ def check_resolution_compatibility_of_sensor_data(self, data, **kwargs):
 
         For a sensor recording instantaneous values, any event frequency is compatible.
         For a sensor recording non-instantaneous values, the event frequency must fit the sensor's event resolution.
-        Currently, only upsampling is supported (e.g. converting hourly events to 15-minute events).
+        Upsampling and downsampling are supported when the inferred resolution and
+        the sensor resolution are multiples of each other.
         """
         required_resolution = data["sensor"].event_resolution
 
@@ -380,7 +381,9 @@ def check_resolution_compatibility_of_sensor_data(self, data, **kwargs):
         # The event frequency is inferred by assuming sequential, equidistant values within a time interval.
         # The event resolution is assumed to be equal to the event frequency.
         inferred_resolution = data["duration"] / len(data["values"])
-        if inferred_resolution % required_resolution != timedelta(hours=0):
+        if inferred_resolution % required_resolution != timedelta(
+            hours=0
+        ) and required_resolution % inferred_resolution != timedelta(hours=0):
             raise ValidationError(
                 f"Resolution of {inferred_resolution} is incompatible with the sensor's required resolution of {required_resolution}."
             )
@@ -407,6 +410,7 @@ def post_load_sequence(self, data: dict, **kwargs) -> dict[str, BeliefsDataFrame
         data = self.possibly_upsample_values(data)
         data = self.possibly_convert_units(data)
         bdf = self.load_bdf(data)
+        bdf = self.possibly_downsample_bdf(bdf, data["sensor"].event_resolution)
 
         # Post-load validation against message type
         _type = data.get("type", None)
@@ -429,7 +433,7 @@ def possibly_convert_units(data):
             data["values"],
             from_unit=data["unit"],
             to_unit=data["sensor"].unit,
-            event_resolution=data["sensor"].event_resolution,
+            event_resolution=data["duration"] / len(data["values"]),
         )
         return data
 
@@ -457,6 +461,21 @@ def possibly_upsample_values(data):
             )
         return data
 
+    @staticmethod
+    def possibly_downsample_bdf(
+        bdf: BeliefsDataFrame, required_resolution: timedelta
+    ) -> BeliefsDataFrame:
+        """
+        Downsample the data if needed, to fit the sensor's resolution.
+        Marshmallow runs this after validation.
+        """
+        if required_resolution == timedelta(hours=0):
+            return bdf
+
+        if bdf.event_resolution < required_resolution:
+            bdf = bdf.resample_events(required_resolution)
+        return bdf
+
     @staticmethod
     def load_bdf(sensor_data: dict) -> BeliefsDataFrame:
         """
@@ -469,7 +488,7 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame:
         sensor = sensor_data["sensor"]
 
         if sensor.event_resolution != timedelta(0) and sensor.get_attribute(
-            "round_datetimes_on_ingestion", True
+            "floor_datetimes_to_resolution", True
         ):
             start = pd.Timestamp(start).floor(sensor.event_resolution)
         elif frequency := sensor.get_attribute("frequency"):

flexmeasures/api/v3_0/tests/test_sensor_data.py

@@ -200,10 +200,10 @@ def test_post_sensor_data_bad_auth(
         ("start", "2021-06-07T00:00:00", "start", "Not a valid aware datetime"),
         (
             "duration",
-            "PT30M",
+            "PT25M",
             "_schema",
-            "Resolution of 0:05:00 is incompatible",
-        ),  # downsampling not supported
+            "Resolution of 0:04:10 is incompatible",
+        ),
         ("unit", "m", "_schema", "Required unit"),
         ("type", "GetSensorDataRequest", "type", "Must be one of"),
     ],
@@ -243,14 +243,40 @@ def test_post_invalid_sensor_data(
 @pytest.mark.parametrize(
     "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True
 )
+@pytest.mark.parametrize(
+    "offclock_start, precise_start, precise_end, values, expected_values",
+    [
+        (
+            "2021-06-08T00:00:40+02:00",
+            "2021-06-08T00:00:00+02:00",
+            "2021-06-08T01:00:00+02:00",
+            [-11.28] * 6,
+            [-11.28] * 6,
+        ),
+        (
+            "2021-06-09T00:00:40+02:00",
+            "2021-06-09T00:00:00+02:00",
+            "2021-06-09T01:00:00+02:00",
+            [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200],
+            [150, 350, 550, 750, 950, 1150],
+        ),
+    ],
+)
 def test_post_non_instantaneous_sensor_data_floor(
-    client, setup_api_test_data, requesting_user
+    client,
+    setup_api_test_data,
+    requesting_user,
+    offclock_start,
+    precise_start,
+    precise_end,
+    values,
+    expected_values,
 ):
-    offclock_start = "2021-06-08T00:00:40+02:00"
-    precise_start = "2021-06-08T00:00:00+02:00"
-    precise_end = "2021-06-08T01:00:00+02:00"
-    post_data = make_sensor_data_request_for_gas_sensor(unit="m³/h")
+    post_data = make_sensor_data_request_for_gas_sensor(
+        num_values=len(values), unit="m³/h"
+    )
     post_data["start"] = offclock_start
+    post_data["values"] = values
     sensor = setup_api_test_data["some gas sensor"]
 
     assert (
@@ -267,13 +293,14 @@ def test_post_non_instantaneous_sensor_data_floor(
     new_data = sensor.search_beliefs(precise_start, precise_end).reset_index()
     assert len(new_data) == 6
     assert list(new_data["event_start"]) == [
-        pd.Timestamp("2021-06-08 00:00:00+02"),
-        pd.Timestamp("2021-06-08 00:10:00+02"),
-        pd.Timestamp("2021-06-08 00:20:00+02"),
-        pd.Timestamp("2021-06-08 00:30:00+02"),
-        pd.Timestamp("2021-06-08 00:40:00+02"),
-        pd.Timestamp("2021-06-08 00:50:00+02"),
+        pd.Timestamp(precise_start),
+        pd.Timestamp(precise_start) + pd.Timedelta(minutes=10),
+        pd.Timestamp(precise_start) + pd.Timedelta(minutes=20),
+        pd.Timestamp(precise_start) + pd.Timedelta(minutes=30),
+        pd.Timestamp(precise_start) + pd.Timedelta(minutes=40),
+        pd.Timestamp(precise_start) + pd.Timedelta(minutes=50),
     ]
+    assert new_data["event_value"].to_list() == expected_values
 
 
 @pytest.mark.parametrize(