Xiaomi: Fix re-processing of some wrongly written files

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/devices/xiaomi/XiaomiSettingsCustomizer.java

@@ -34,6 +34,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -188,8 +189,7 @@ public class XiaomiSettingsCustomizer implements DeviceSpecificSettingsCustomize
                         continue;
                     }
 
-                    final byte[] fileIdBytes = Arrays.copyOfRange(data, 0, 7);
+                    final XiaomiActivityFileId fileId = XiaomiActivityFileId.from(data);
-                    final XiaomiActivityFileId fileId = XiaomiActivityFileId.from(fileIdBytes);
 
                     final XiaomiActivityParser activityParser = XiaomiActivityParser.create(fileId);
                     if (activityParser == null) {
@@ -198,7 +198,9 @@ public class XiaomiSettingsCustomizer implements DeviceSpecificSettingsCustomize
                     }
 
                     try {
-                        if (activityParser.parse(context, device, fileId, data)) {
+                        // Some files may have been wrongly written, see javadoc for fixAndWrap
+                        final byte[] fixedData = XiaomiActivityParser.fixAndWrap(data).array();
+                        if (activityParser.parse(context, device, fileId, fixedData)) {
                             LOG.info("Successfully parsed {}", fileId);
                         } else {
                             LOG.warn("Failed to parse {}", fileId);

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/xiaomi/activity/XiaomiActivityParser.java

@@ -23,6 +23,8 @@ import androidx.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.List;
 
 import de.greenrobot.dao.query.QueryBuilder;
@@ -33,6 +35,7 @@ import nodomain.freeyourgadget.gadgetbridge.entities.Device;
 import nodomain.freeyourgadget.gadgetbridge.entities.User;
 import nodomain.freeyourgadget.gadgetbridge.impl.GBDevice;
 import nodomain.freeyourgadget.gadgetbridge.model.ActivityKind;
+import nodomain.freeyourgadget.gadgetbridge.service.btle.BLETypeConversions;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.impl.ManualSamplesParser;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.impl.DailyDetailsParser;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.impl.DailySummaryParser;
@@ -40,6 +43,7 @@ import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.impl
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.impl.SleepStagesParser;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.impl.WorkoutGpsParser;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.impl.WorkoutSummaryParser;
+import nodomain.freeyourgadget.gadgetbridge.util.CheckSums;
 
 public abstract class XiaomiActivityParser {
     private static final Logger LOG = LoggerFactory.getLogger(XiaomiActivityParser.class);
@@ -133,4 +137,34 @@ public abstract class XiaomiActivityParser {
     public static boolean validData(final byte[] header, final int i) {
         return (header[i / 8] & (1 << (7 - (i % 8)))) != 0;
     }
+
+    /**
+     * If the CRC32 is not valid, we're missing 1 header padding byte due to a previous bug.
+     * This previous version also did not include the CRC at the end.
+     * More info: <a href="https://codeberg.org/Freeyourgadget/Gadgetbridge/issues/3916">#3916</a>
+     */
+    public static ByteBuffer fixAndWrap(final byte[] data) {
+        final int arrCrc32 = CheckSums.getCRC32(data, 0, data.length - 4);
+        final int expectedCrc32 = BLETypeConversions.toUint32(data, data.length - 4);
+
+        if (arrCrc32 == expectedCrc32) {
+            return ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN);
+        }
+
+        LOG.warn("Invalid CRC found, fixing and wrapping");
+
+        // Unfortunately in this case we can't distinguish a corrupted file from one where we accidentally discarded the
+        // crc. We assume the latter, since we check the crc before writing it to storage. We then reconstruct the file
+        // by adding the missing padding byte + append a newly computed crc
+        final ByteBuffer buf = ByteBuffer.allocate(data.length + 1 + 4).order(ByteOrder.LITTLE_ENDIAN);
+        buf.put(data, 0, 7); // fileId
+        buf.put((byte) 0); // missing padding byte
+        buf.put(data, 7, data.length - 7);
+
+        final int newCrc32 = CheckSums.getCRC32(buf.array(), 0, buf.limit() - 4);
+        buf.putInt(newCrc32);
+        buf.flip();
+
+        return buf;
+    }
 }

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/xiaomi/activity/XiaomiActivityTrackProvider.java

@@ -53,7 +53,8 @@ public class XiaomiActivityTrackProvider implements ActivityTrackProvider {
         try {
             final WorkoutGpsParser workoutGpsParser = new WorkoutGpsParser();
             final XiaomiActivityFileId fileId = XiaomiActivityFileId.from(detailsBytes);
-            return workoutGpsParser.getActivityTrack(fileId, detailsBytes);
+            final byte[] fixedBytes = XiaomiActivityParser.fixAndWrap(detailsBytes).array();
+            return workoutGpsParser.getActivityTrack(fileId, fixedBytes);
         } catch (final Exception e) {
             LOG.error("Failed to parse bytes from {}", inputFile, e);
             return new GpxActivityTrackProvider().getActivityTrack(summary);

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/xiaomi/activity/impl/WorkoutGpsParser.java

@@ -120,6 +120,7 @@ public class WorkoutGpsParser extends XiaomiActivityParser {
         return activityTrack;
     }
 
+    @Override
     public boolean parse(final Context context, final GBDevice gbDevice, final XiaomiActivityFileId fileId, final byte[] bytes) {
         final ActivityTrack activityTrack = getActivityTrack(fileId, bytes);
         if (activityTrack == null) {

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/xiaomi/activity/impl/WorkoutSummaryParser.java

@@ -112,10 +112,8 @@ import nodomain.freeyourgadget.gadgetbridge.entities.User;
 import nodomain.freeyourgadget.gadgetbridge.impl.GBDevice;
 import nodomain.freeyourgadget.gadgetbridge.model.ActivityKind;
 import nodomain.freeyourgadget.gadgetbridge.model.ActivitySummaryParser;
-import nodomain.freeyourgadget.gadgetbridge.service.btle.BLETypeConversions;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.XiaomiActivityFileId;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.XiaomiActivityParser;
-import nodomain.freeyourgadget.gadgetbridge.util.CheckSums;
 import nodomain.freeyourgadget.gadgetbridge.util.GB;
 
 @SuppressWarnings("NonStrictComparisonCanBeEquality")
@@ -132,19 +130,14 @@ public class WorkoutSummaryParser extends XiaomiActivityParser implements Activi
         summary.setRawSummaryData(bytes);
 
         try {
-            summary = parseBinaryData(summary, true);
+            final ByteBuffer buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN);
+            updateSummaryFromData(summary, buf, true);
         } catch (final Exception e) {
             LOG.error("Failed to parse workout summary", e);
             GB.toast(context, "Failed to parse workout summary", Toast.LENGTH_LONG, GB.ERROR, e);
             return false;
         }
 
-        // parseBinaryData may return null in case the version is not supported
-        if (summary == null) {
-            LOG.warn("summary is null - should never happen {}", fileId);
-            return false;
-        }
-
         summary.setSummaryData(null); // remove json before saving to database
 
         try (DBHandler dbHandler = GBApplication.acquireDB()) {
@@ -174,24 +167,12 @@ public class WorkoutSummaryParser extends XiaomiActivityParser implements Activi
             return summary;
         }
 
-        final int arrCrc32 = CheckSums.getCRC32(data, 0, data.length - 4);
+        return updateSummaryFromData(summary, fixAndWrap(data), forDetails);
-        final int expectedCrc32 = BLETypeConversions.toUint32(data, data.length - 4);
+    }
-
-        final ByteBuffer buf;
-        if (arrCrc32 != expectedCrc32) {
-            // If the CRC32 is not valid, we're missing 1 header padding byte due to a previous bug
-            // This previous version also did not include the CRC at the end
-            // More info: https://codeberg.org/Freeyourgadget/Gadgetbridge/issues/3916
-            buf = ByteBuffer.allocate(data.length + 1).order(ByteOrder.LITTLE_ENDIAN);
-            buf.put(data, 0, 7);
-            buf.put((byte) 0);
-            buf.put(data, 7, data.length - 7);
-            buf.flip();
-        } else {
-            // Valid full file, skip crc
-            buf = ByteBuffer.wrap(data, 0, data.length - 4).order(ByteOrder.LITTLE_ENDIAN);
-        }
 
+    private BaseActivitySummary updateSummaryFromData(final BaseActivitySummary summary,
+                                                      final ByteBuffer buf,
+                                                      final boolean forDetails) {
         final XiaomiActivityFileId fileId = XiaomiActivityFileId.from(buf);
 
         final byte fileIdPadding = buf.get();