Xiaomi: Improve sleep parsing

- Some devices send a random int 0, which would prevent sleep stage
  parsing
- Some devices send the details as a file of type summary, but same
  structure
- It is still not stable for all devices

Thanks to @opcode for the parsing logic

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/xiaomi/activity/XiaomiActivityParser.java

@@ -111,11 +111,7 @@ public abstract class XiaomiActivityParser {
 
                 break;
             case ACTIVITY_SLEEP:
-                if (fileId.getDetailType() == XiaomiActivityFileId.DetailType.DETAILS) {
+                return new SleepDetailsParser();
-                    return new SleepDetailsParser();
-                }
-
-                break;
         }
 
         return null;

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/xiaomi/activity/impl/SleepDetailsParser.java

@@ -16,12 +16,12 @@
     along with this program.  If not, see <https://www.gnu.org/licenses/>. */
 package nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.impl;
 
-import android.util.Log;
 import android.widget.Toast;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.nio.BufferUnderflowException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.ArrayList;
@@ -48,17 +48,23 @@ public class SleepDetailsParser extends XiaomiActivityParser {
 
     @Override
     public boolean parse(final XiaomiSupport support, final XiaomiActivityFileId fileId, final byte[] bytes) {
-        if (fileId.getVersion() != 2) {
+        // Seems to come both as DetailType.DETAILS (version 2) and DetailType.SUMMARY (version 4)
+        if (fileId.getVersion() < 2 || fileId.getVersion() > 4) {
             LOG.warn("Unknown sleep details version {}", fileId.getVersion());
             return false;
         }
 
         final ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
-        buf.get(); // header ? 0xF0
+        final byte header = buf.get();
 
         final int isAwake = buf.get() & 0xff; // 0/1 - more correctly this would be !isSleepFinish
         final int bedTime = buf.getInt();
         final int wakeupTime = buf.getInt();
+        int sleepQuality = -1;
+        if (fileId.getVersion() >= 4) {
+            sleepQuality = buf.get() & 0xff;
+        }
+
         LOG.debug("Sleep sample: bedTime: {}, wakeupTime: {}, isAwake: {}", bedTime, wakeupTime, isAwake);
 
         final List<XiaomiSleepTimeSample> summaries = new ArrayList<>();
@@ -68,10 +74,8 @@ public class SleepDetailsParser extends XiaomiActivityParser {
         sample.setWakeupTime(wakeupTime * 1000L);
         sample.setIsAwake(isAwake == 1);
 
-        // SleepAssistItemInfo 2x
+        // Heart rate samples
-        //  - 0: Heart rate samples
+        if ((header & (1 << 4)) != 0) {
-        //  - 1: Sp02 samples
-        for (int i = 0; i < 2; i++) {
             final int unit = buf.getShort(); // Time unit (i.e sample rate)
             final int count = buf.getShort();
             final int firstRecordTime = buf.getInt();
@@ -81,86 +85,123 @@ public class SleepDetailsParser extends XiaomiActivityParser {
             buf.position(buf.position() + count);
         }
 
-        final List<XiaomiSleepStageSample> stages = new ArrayList<>();
+        // SpO2 samples
+        if ((header & (1 << 3)) != 0) {
+            final int unit = buf.getShort(); // Time unit (i.e sample rate)
+            final int count = buf.getShort();
+            final int firstRecordTime = buf.getInt();
 
-        while (buf.remaining() >= 17 && buf.getInt() == 0xFFFCFAFB) {
+            // Skip count samples - each sample is a u8
-            final int headerLen = buf.get() & 0xFF; // this seems to always be 17
+            //   timestamp of each sample is firstRecordTime + (unit * index)
-
+            buf.position(buf.position() + count);
-            // This timestamp is kind of weird, is seems to sometimes be in seconds
-            // and other times in nanoseconds. Message types 16 and 17 are in seconds
-            final long ts = buf.getLong();
-            final int unk = buf.get() & 0xFF;
-            final int type = buf.get() & 0xFF;
-
-            final int dataLen = ((buf.get() & 0xFF) << 8) | (buf.get() & 0xFF);
-
-            final byte[] data = new byte[dataLen];
-            buf.get(data);
-
-            final ByteBuffer dataBuf = ByteBuffer.wrap(data).order(ByteOrder.BIG_ENDIAN);
-
-//            Known types:
-//             - acc_unk = 0,
-//             - ppg_unk = 1,
-//             - fall_asleep = 2,
-//             - wake_up = 3,
-//             - switch_ts_unk1 = 12,
-//             - switch_ts_unk2 = 13,
-//             - Summary = 16,
-//             - Stages = 17
-
-            if (type == 16) {
-                final int data_0 = dataBuf.get() & 0xFF;
-                final int sleep_index = data_0 >> 4;
-                final int wake_count = data_0 & 0x0F;
-
-                final int sleep_duration = dataBuf.getShort() & 0xFFFF;
-                final int wake_duration  = dataBuf.getShort() & 0xFFFF;
-                final int light_duration = dataBuf.getShort() & 0xFFFF;
-                final int rem_duration   = dataBuf.getShort() & 0xFFFF;
-                final int deep_duration  = dataBuf.getShort() & 0xFFFF;
-
-                final int data_1 = dataBuf.get() & 0xFF;
-                final boolean has_rem = (data_1 >> 4) == 1;
-                final boolean has_stage = (data_1 >> 2) == 1;
-
-                // Could probably be an "awake" duration after sleep
-                final int unk_duration_minutes = dataBuf.get() & 0xFF;
-
-                if (sample == null) {
-                    sample = new XiaomiSleepTimeSample();
-                }
-
-                sample.setTimestamp(bedTime * 1000L);
-                sample.setWakeupTime(wakeupTime * 1000L);
-                sample.setTotalDuration(sleep_duration);
-                sample.setDeepSleepDuration(deep_duration);
-                sample.setLightSleepDuration(light_duration);
-                sample.setRemSleepDuration(rem_duration);
-                sample.setAwakeDuration(wake_duration);
-
-                summaries.add(sample);
-                sample = null;
-            }
-            else if (type == 17) { // Stages
-                long currentTime = ts * 1000;
-                for (int i = 0; i < dataLen / 2; i++) {
-                    // when the change to the phase occurs
-                    final int val = dataBuf.getShort() & 0xFFFF;
-
-                    final int stage = val >> 12;
-                    final int offsetMinutes = val & 0xFFF;
-
-                    final XiaomiSleepStageSample stageSample = new XiaomiSleepStageSample();
-                    stageSample.setTimestamp(currentTime);
-                    stageSample.setStage(decodeStage(stage));
-                    stages.add(stageSample);
-
-                    currentTime += offsetMinutes * 60000;
-                }
-            }
         }
 
+        // snore samples
+        if (fileId.getVersion() >= 3 && (header & (1 << 2)) != 0) {
+            final int unit = buf.getShort(); // Time unit (i.e sample rate)
+            final int count = buf.getShort();
+            final int firstRecordTime = buf.getInt();
+
+            // Skip count samples - each sample is a float
+            //   timestamp of each sample is firstRecordTime + (unit * index)
+            buf.position(buf.position() + count * 4);
+        }
+
+        final List<XiaomiSleepStageSample> stages = new ArrayList<>();
+
+        // Do not crash if we face a buffer underflow, as the next parsing is not 100% fool-proof,
+        // and we still want to persist whatever we got so far
+        boolean stagesParseFailed = false;
+        try {
+            // FIXME: Sometimes there's a random zero here..?
+            if (buf.getInt() != 0) {
+                // If it wasn't a zero, rewind
+                buf.position(buf.position() - 4);
+            }
+
+            while (buf.remaining() >= 17 && buf.getInt() == 0xFFFCFAFB) {
+                final int headerLen = buf.get() & 0xFF; // this seems to always be 17
+
+                // This timestamp is kind of weird, is seems to sometimes be in seconds
+                // and other times in nanoseconds. Message types 16 and 17 are in seconds
+                final long ts = buf.getLong();
+                final int unk = buf.get() & 0xFF;
+                final int type = buf.get() & 0xFF;
+
+                final int dataLen = ((buf.get() & 0xFF) << 8) | (buf.get() & 0xFF);
+
+                final byte[] data = new byte[dataLen];
+                buf.get(data);
+
+                final ByteBuffer dataBuf = ByteBuffer.wrap(data).order(ByteOrder.BIG_ENDIAN);
+
+                // Known types:
+                //  - acc_unk = 0,
+                //  - ppg_unk = 1,
+                //  - fall_asleep = 2,
+                //  - wake_up = 3,
+                //  - switch_ts_unk1 = 12,
+                //  - switch_ts_unk2 = 13,
+                //  - Summary = 16,
+                //  - Stages = 17
+
+                if (type == 16) {
+                    final int data_0 = dataBuf.get() & 0xFF;
+                    final int sleep_index = data_0 >> 4;
+                    final int wake_count = data_0 & 0x0F;
+
+                    final int sleep_duration = dataBuf.getShort() & 0xFFFF;
+                    final int wake_duration  = dataBuf.getShort() & 0xFFFF;
+                    final int light_duration = dataBuf.getShort() & 0xFFFF;
+                    final int rem_duration   = dataBuf.getShort() & 0xFFFF;
+                    final int deep_duration  = dataBuf.getShort() & 0xFFFF;
+
+                    final int data_1 = dataBuf.get() & 0xFF;
+                    final boolean has_rem = (data_1 >> 4) == 1;
+                    final boolean has_stage = (data_1 >> 2) == 1;
+
+                    // Could probably be an "awake" duration after sleep
+                    final int unk_duration_minutes = dataBuf.get() & 0xFF;
+
+                    if (sample == null) {
+                        sample = new XiaomiSleepTimeSample();
+                    }
+
+                    sample.setTimestamp(bedTime * 1000L);
+                    sample.setWakeupTime(wakeupTime * 1000L);
+                    sample.setTotalDuration(sleep_duration);
+                    sample.setDeepSleepDuration(deep_duration);
+                    sample.setLightSleepDuration(light_duration);
+                    sample.setRemSleepDuration(rem_duration);
+                    sample.setAwakeDuration(wake_duration);
+
+                    // FIXME: This is an array, but we end up persisting only the last sample, since
+                    // the timestamp is the primary key
+                    summaries.add(sample);
+                    sample = null;
+                }
+                else if (type == 17) { // Stages
+                    long currentTime = ts * 1000;
+                    for (int i = 0; i < dataLen / 2; i++) {
+                        // when the change to the phase occurs
+                        final int val = dataBuf.getShort() & 0xFFFF;
+
+                        final int stage = val >> 12;
+                        final int offsetMinutes = val & 0xFFF;
+
+                        final XiaomiSleepStageSample stageSample = new XiaomiSleepStageSample();
+                        stageSample.setTimestamp(currentTime);
+                        stageSample.setStage(decodeStage(stage));
+                        stages.add(stageSample);
+
+                        currentTime += offsetMinutes * 60000;
+                    }
+                }
+            }
+        } catch (final BufferUnderflowException e) {
+            LOG.warn("Buffer underflow while parsing sleep stages...", e);
+            stagesParseFailed = true;
+        }
 
         // save all the samples that we got
         try (DBHandler handler = GBApplication.acquireDB()) {
@@ -169,7 +210,6 @@ public class SleepDetailsParser extends XiaomiActivityParser {
 
             final XiaomiSleepTimeSampleProvider sampleProvider = new XiaomiSleepTimeSampleProvider(gbDevice, session);
 
-
             for (final XiaomiSleepTimeSample summary : summaries) {
                 summary.setDevice(DBHelper.getDevice(gbDevice, session));
                 summary.setUser(DBHelper.getUser(session));
@@ -195,28 +235,32 @@ public class SleepDetailsParser extends XiaomiActivityParser {
             return false;
         }
 
-        // Save the sleep stage samples
+        if (!stagesParseFailed && !stages.isEmpty()) {
-        try (DBHandler handler = GBApplication.acquireDB()) {
+            LOG.debug("Persisting {} sleep stage samples", stages.size());
-            final DaoSession session = handler.getDaoSession();
-            final GBDevice gbDevice = support.getDevice();
-            final Device device = DBHelper.getDevice(gbDevice, session);
-            final User user = DBHelper.getUser(session);
 
-            final XiaomiSleepStageSampleProvider sampleProvider = new XiaomiSleepStageSampleProvider(gbDevice, session);
+            // Save the sleep stage samples
+            try (DBHandler handler = GBApplication.acquireDB()) {
+                final DaoSession session = handler.getDaoSession();
+                final GBDevice gbDevice = support.getDevice();
+                final Device device = DBHelper.getDevice(gbDevice, session);
+                final User user = DBHelper.getUser(session);
 
-            for (final XiaomiSleepStageSample stageSample : stages) {
+                final XiaomiSleepStageSampleProvider sampleProvider = new XiaomiSleepStageSampleProvider(gbDevice, session);
-                stageSample.setDevice(device);
+
-                stageSample.setUser(user);
+                for (final XiaomiSleepStageSample stageSample : stages) {
+                    stageSample.setDevice(device);
+                    stageSample.setUser(user);
+                }
+
+                sampleProvider.addSamples(stages);
+            } catch (final Exception e) {
+                GB.toast(support.getContext(), "Error saving sleep stage samples", Toast.LENGTH_LONG, GB.ERROR);
+                LOG.error("Error saving sleep stage samples", e);
+                return false;
             }
-
-            sampleProvider.addSamples(stages);
-        } catch (final Exception e) {
-            GB.toast(support.getContext(), "Error saving sleep stage samples", Toast.LENGTH_LONG, GB.ERROR);
-            LOG.error("Error saving sleep stage samples", e);
-            return false;
         }
 
-        return true;
+        return stagesParseFailed;
     }
 
     static private int decodeStage(int rawStage) {