db refactoring: remove raw activity kind field from pebble health minute data, add a blob column for raw undecoded health minute data.

also:
- change column order for all pebble related minute data tables (mandatory stuff first, then custom columns, for easier addition of new columns)
- remove unused code

TODO:
- fix column order in Mi Band table also

GBDaoGenerator/src/nodomain/freeyourgadget/gadgetbridge/daogen/GBDaoGenerator.java

@@ -34,7 +34,7 @@ public class GBDaoGenerator {
     private static final String VALID_BY_DATE = MODEL_PACKAGE + ".ValidByDate";
 
     public static void main(String[] args) throws Exception {
-        Schema schema = new Schema(11, MAIN_PACKAGE + ".entities");
+        Schema schema = new Schema(12, MAIN_PACKAGE + ".entities");
 
         addActivityDescription(schema);
 
@@ -131,10 +131,11 @@ public class GBDaoGenerator {
     }
 
     private static Entity addMiBandActivitySample(Schema schema, Entity user, Entity device) {
-//        public GBActivitySample(SampleProvider provider, int timestamp, int intensity, int steps, int type, int customValue) {
         Entity activitySample = addEntity(schema, "MiBandActivitySample");
         addCommonActivitySampleProperties("AbstractActivitySample", activitySample, user, device);
-        addDefaultActivitySampleAttributes(activitySample);
+        activitySample.addIntProperty("rawIntensity").notNull();
+        activitySample.addIntProperty("steps").notNull();
+        activitySample.addIntProperty("rawKind").notNull();
         addCommonActivitySampleProperties2(activitySample, user, device);
         addHeartRateProperties(activitySample);
         return activitySample;
@@ -145,11 +146,12 @@ public class GBDaoGenerator {
     }
 
     private static Entity addPebbleHealthActivitySample(Schema schema, Entity user, Entity device) {
-//        public GBActivitySample(SampleProvider provider, int timestamp, int intensity, int steps, int type, int customValue) {
         Entity activitySample = addEntity(schema, "PebbleHealthActivitySample");
-        addCommonActivitySampleProperties("AbstractActivitySample", activitySample, user, device);
+        addCommonActivitySampleProperties("AbstractPebbleHealthActivitySample", activitySample, user, device);
-        addDefaultActivitySampleAttributes(activitySample);
         addCommonActivitySampleProperties2(activitySample, user, device);
+        activitySample.addByteArrayProperty("rawPebbleHealthData");
+        activitySample.addIntProperty("rawIntensity").notNull();
+        activitySample.addIntProperty("steps").notNull();
         return activitySample;
     }
 
@@ -178,16 +180,16 @@ public class GBDaoGenerator {
     private static Entity addPebbleMisfitActivitySample(Schema schema, Entity user, Entity device) {
         Entity activitySample = addEntity(schema, "PebbleMisfitSample");
         addCommonActivitySampleProperties("AbstractPebbleMisfitActivitySample", activitySample, user, device);
-        activitySample.addIntProperty("rawPebbleMisfitSample").notNull();
         addCommonActivitySampleProperties2(activitySample, user, device);
+        activitySample.addIntProperty("rawPebbleMisfitSample").notNull();
         return activitySample;
     }
 
     private static Entity addPebbleMorpheuzActivitySample(Schema schema, Entity user, Entity device) {
         Entity activitySample = addEntity(schema, "PebbleMorpheuzSample");
         addCommonActivitySampleProperties("AbstractPebbleMorpheuzActivitySample", activitySample, user, device);
-        activitySample.addIntProperty("rawIntensity").notNull();
         addCommonActivitySampleProperties2(activitySample, user, device);
+        activitySample.addIntProperty("rawIntensity").notNull();
         return activitySample;
     }
 
@@ -224,12 +226,6 @@ public class GBDaoGenerator {
         throw new IllegalArgumentException("Property " + propertyName + " not found in Entity " + entity.getClassName());
     }
 
-    private static void addDefaultActivitySampleAttributes(Entity activitySample) {
-        activitySample.addIntProperty("rawIntensity").notNull();
-        activitySample.addIntProperty("steps").notNull();
-        activitySample.addIntProperty("rawKind").notNull();
-    }
-
     private static Entity addEntity(Schema schema, String className) {
         Entity entity = schema.addEntity(className);
         entity.addImport("de.greenrobot.dao.AbstractDao");

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/devices/AbstractSampleProvider.java

@@ -61,18 +61,6 @@ public abstract class AbstractSampleProvider<T extends AbstractActivitySample> i
         }
     }
 
-    @Override
-    public int fetchLatestTimestamp() {
-        QueryBuilder<T> qb = getSampleDao().queryBuilder();
-        qb.orderDesc(getTimestampSampleProperty());
-        qb.limit(1);
-        List<T> list = qb.build().list();
-        if (list.size() >= 1) {
-            return list.get(0).getTimestamp();
-        }
-        return -1;
-    }
-
     @Override
     public void addGBActivitySample(T activitySample) {
         getSampleDao().insertOrReplace(activitySample);
@@ -83,22 +71,6 @@ public abstract class AbstractSampleProvider<T extends AbstractActivitySample> i
         getSampleDao().insertOrReplaceInTx(activitySamples);
     }
 
-    public void changeStoredSamplesType(int timestampFrom, int timestampTo, int kind) {
-        List<T> samples = getAllActivitySamples(timestampFrom, timestampTo);
-        for (T sample : samples) {
-            sample.setRawKind(kind);
-        }
-        getSampleDao().updateInTx(samples);
-    }
-
-    public void changeStoredSamplesType(int timestampFrom, int timestampTo, int fromKind, int toKind) {
-        List<T> samples = getGBActivitySamples(timestampFrom, timestampTo, fromKind);
-        for (T sample : samples) {
-            sample.setRawKind(toKind);
-        }
-        getSampleDao().updateInTx(samples);
-    }
-
     protected List<T> getGBActivitySamples(int timestamp_from, int timestamp_to, int activityType) {
         if (getRawKindSampleProperty() == null && activityType != ActivityKind.TYPE_ALL) {
             // if we do not have a raw kind property we cannot query anything else then TYPE_ALL

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/devices/SampleProvider.java

@@ -28,12 +28,6 @@ public interface SampleProvider<T extends AbstractActivitySample> {
 
     List<T> getSleepSamples(int timestamp_from, int timestamp_to);
 
-    void changeStoredSamplesType(int timestampFrom, int timestampTo, int kind);
-
-    void changeStoredSamplesType(int timestampFrom, int timestampTo, int fromKind, int toKind);
-
-    int fetchLatestTimestamp();
-
     void addGBActivitySample(T activitySample);
 
     void addGBActivitySamples(T[] activitySamples);

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/devices/UnknownDeviceCoordinator.java

@@ -7,7 +7,6 @@ import android.net.Uri;
 import java.util.List;
 
 import nodomain.freeyourgadget.gadgetbridge.activities.ControlCenter;
-import nodomain.freeyourgadget.gadgetbridge.database.DBHandler;
 import nodomain.freeyourgadget.gadgetbridge.entities.AbstractActivitySample;
 import nodomain.freeyourgadget.gadgetbridge.entities.DaoSession;
 import nodomain.freeyourgadget.gadgetbridge.impl.GBDevice;
@@ -49,19 +48,6 @@ public class UnknownDeviceCoordinator extends AbstractDeviceCoordinator {
             return null;
         }
 
-        @Override
-        public void changeStoredSamplesType(int timestampFrom, int timestampTo, int kind) {
-        }
-
-        @Override
-        public void changeStoredSamplesType(int timestampFrom, int timestampTo, int fromKind, int toKind) {
-        }
-
-        @Override
-        public int fetchLatestTimestamp() {
-            return 0;
-        }
-
         @Override
         public void addGBActivitySample(AbstractActivitySample activitySample) {
         }

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/entities/AbstractPebbleHealthActivitySample.java

@@ -0,0 +1,19 @@
+package nodomain.freeyourgadget.gadgetbridge.entities;
+
+import nodomain.freeyourgadget.gadgetbridge.model.ActivityKind;
+
+public abstract class AbstractPebbleHealthActivitySample extends AbstractActivitySample {
+    abstract public byte[] getRawPebbleHealthData();
+
+    private transient int rawActivityKind = ActivityKind.TYPE_UNKNOWN;
+
+    @Override
+    public int getRawKind() {
+        return rawActivityKind;
+    }
+
+    @Override
+    public void setRawKind(int kind) {
+        this.rawActivityKind = kind;
+    }
+}

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/pebble/AppMessageHandlerMisfit.java

@@ -83,7 +83,7 @@ public class AppMessageHandlerMisfit extends AppMessageHandler {
                         Long deviceId = DBHelper.getDevice(getDevice(), db.getDaoSession()).getId();
                         for (int i = 0; i < samples; i++) {
                             short sample = buf.getShort();
-                            misfitSamples[i] = new PebbleMisfitSample(null, timestamp + i * 60, sample & 0xffff, userId, deviceId);
+                            misfitSamples[i] = new PebbleMisfitSample(null, timestamp + i * 60, userId, deviceId, sample & 0xffff);
                             misfitSamples[i].setProvider(sampleProvider);
                             int steps = misfitSamples[i].getSteps();
                             totalSteps += steps;

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/pebble/AppMessageHandlerMorpheuz.java

@@ -114,7 +114,7 @@ public class AppMessageHandlerMorpheuz extends AppMessageHandler {
                                 Long userId = DBHelper.getUser(db.getDaoSession()).getId();
                                 Long deviceId = DBHelper.getDevice(getDevice(), db.getDaoSession()).getId();
                                 PebbleMorpheuzSampleProvider sampleProvider = new PebbleMorpheuzSampleProvider(getDevice(), db.getDaoSession());
-                                PebbleMorpheuzSample sample = new PebbleMorpheuzSample(null, recording_base_timestamp + index * 600, intensity, userId, deviceId);
+                                PebbleMorpheuzSample sample = new PebbleMorpheuzSample(null, recording_base_timestamp + index * 600, userId, deviceId, intensity);
                                 sample.setProvider(sampleProvider);
                                 sampleProvider.addGBActivitySample(sample);
                             } catch (Exception e) {

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/pebble/DatalogSessionHealthSteps.java

@@ -13,7 +13,6 @@ import nodomain.freeyourgadget.gadgetbridge.database.DBHelper;
 import nodomain.freeyourgadget.gadgetbridge.devices.pebble.PebbleHealthSampleProvider;
 import nodomain.freeyourgadget.gadgetbridge.entities.PebbleHealthActivitySample;
 import nodomain.freeyourgadget.gadgetbridge.impl.GBDevice;
-import nodomain.freeyourgadget.gadgetbridge.model.ActivityKind;
 import nodomain.freeyourgadget.gadgetbridge.util.GB;
 
 public class DatalogSessionHealthSteps extends DatalogSessionPebbleHealth {
@@ -85,10 +84,11 @@ public class DatalogSessionHealthSteps extends DatalogSessionPebbleHealth {
                 samples[j] = new PebbleHealthActivitySample(
                         null,
                         stepsRecord.timestamp,
+                        userId, deviceId,
+                        null, // raw data here
                         stepsRecord.intensity,
-                        stepsRecord.steps,
+                        stepsRecord.steps
-                        sampleProvider.toRawActivityKind(ActivityKind.TYPE_ACTIVITY),
+                );
-                        userId, deviceId);
                 samples[j].setProvider(sampleProvider);
             }