Garmin protocol: add support for AGPS data retrieval

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/garmin/ProtocolBufferHandler.java

@@ -24,11 +24,13 @@ import nodomain.freeyourgadget.gadgetbridge.externalevents.gps.GBLocationService
 import nodomain.freeyourgadget.gadgetbridge.model.CannedMessagesSpec;
 import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiCalendarService;
 import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiCore;
+import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiDataTransferService;
 import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiDeviceStatus;
 import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiFindMyWatch;
 import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiHttpService;
 import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiSmartProto;
 import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiSmsNotification;
+import nodomain.freeyourgadget.gadgetbridge.service.devices.garmin.http.DataTransferHandler;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.garmin.http.HttpHandler;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.garmin.messages.GFDIMessage;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.garmin.messages.ProtobufMessage;
@@ -45,12 +47,16 @@ public class ProtocolBufferHandler implements MessageHandler {
     private final Map<Integer, ProtobufFragment> chunkedFragmentsMap;
     private final int maxChunkSize = 375; //tested on Vívomove Style
     private int lastProtobufRequestId;
+    private final HttpHandler httpHandler;
+    private final DataTransferHandler dataTransferHandler;
 
     private final Map<GdiSmsNotification.SmsNotificationService.CannedListType, String[]> cannedListTypeMap = new HashMap<>();
 
     public ProtocolBufferHandler(GarminSupport deviceSupport) {
         this.deviceSupport = deviceSupport;
         chunkedFragmentsMap = new HashMap<>();
+        httpHandler = new HttpHandler(deviceSupport);
+        dataTransferHandler = new DataTransferHandler();
     }
 
     private int getNextProtobufRequestId() {
@@ -91,12 +97,19 @@ public class ProtocolBufferHandler implements MessageHandler {
                 return prepareProtobufResponse(processProtobufSmsNotificationMessage(smart.getSmsNotificationService()), message.getRequestId());
             }
             if (smart.hasHttpService()) {
-                final GdiHttpService.HttpService response = HttpHandler.handle(smart.getHttpService());
+                final GdiHttpService.HttpService response = httpHandler.handle(smart.getHttpService());
                 if (response == null) {
                     return null;
                 }
                 return prepareProtobufResponse(GdiSmartProto.Smart.newBuilder().setHttpService(response).build(), message.getRequestId());
             }
+            if (smart.hasDataTransferService()) {
+                final GdiDataTransferService.DataTransferService response = dataTransferHandler.handle(smart.getDataTransferService(), message.getRequestId());
+                if (response == null) {
+                    return null;
+                }
+                return prepareProtobufResponse(GdiSmartProto.Smart.newBuilder().setDataTransferService(response).build(), message.getRequestId());
+            }
             if (smart.hasDeviceStatusService()) {
                 processed = true;
                 processProtobufDeviceStatusResponse(smart.getDeviceStatusService());
@@ -116,6 +129,9 @@ public class ProtocolBufferHandler implements MessageHandler {
     private ProtobufMessage processIncoming(ProtobufStatusMessage statusMessage) {
         LOG.info("Processing protobuf status message #{}@{}:  status={}, error={}", statusMessage.getRequestId(), statusMessage.getDataOffset(), statusMessage.getProtobufChunkStatus(), statusMessage.getProtobufStatusCode());
         //TODO: check status and react accordingly, right now we blindly proceed to next chunk
+        if (statusMessage.isOK()) {
+            DataTransferHandler.onDataSuccessfullyReceived(statusMessage.getRequestId());
+        }
         if (chunkedFragmentsMap.containsKey(statusMessage.getRequestId()) && statusMessage.isOK()) {
             final ProtobufFragment protobufFragment = chunkedFragmentsMap.get(statusMessage.getRequestId());
             LOG.debug("Protobuf message #{} found in queue: {}", statusMessage.getRequestId(), GB.hexdump(protobufFragment.fragmentBytes));

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/garmin/http/DataTransferHandler.java

@@ -0,0 +1,137 @@
+package nodomain.freeyourgadget.gadgetbridge.service.devices.garmin.http;
+
+import com.google.protobuf.ByteString;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiDataTransferService;
+
+public class DataTransferHandler {
+    private static final Logger LOG = LoggerFactory.getLogger(DataTransferHandler.class);
+    private static final AtomicInteger idCounter = new AtomicInteger(0);
+    private static final Map<Integer, Data> dataById = new HashMap<>();
+    private static final Map<Integer, RequestInfo> requestInfoById = new HashMap<>();
+
+    public GdiDataTransferService.DataTransferService handle(
+            final GdiDataTransferService.DataTransferService dataTransferService,
+            final int requestId
+    ) {
+        if (dataTransferService.hasDataDownloadRequest()) {
+            final GdiDataTransferService.DataTransferService.DataDownloadResponse dataDownloadResponse
+                    = handleDataDownloadRequest(dataTransferService.getDataDownloadRequest(), requestId);
+            if (dataDownloadResponse != null) {
+                return GdiDataTransferService.DataTransferService.newBuilder()
+                        .setDataDownloadResponse(dataDownloadResponse)
+                        .build();
+            }
+            return null;
+        }
+
+        LOG.warn("Unsupported data transfer service request: {}", dataTransferService);
+
+        return null;
+    }
+
+    public GdiDataTransferService.DataTransferService.DataDownloadResponse handleDataDownloadRequest(
+            final GdiDataTransferService.DataTransferService.DataDownloadRequest dataDownloadRequest,
+            final int requestId
+    ) {
+        final int dataId = dataDownloadRequest.getId();
+        final int offset = dataDownloadRequest.getOffset();
+        LOG.debug("Received data download request (id: {}, offset: {})", dataId, offset);
+        final Data data = dataById.get(dataId);
+        if (data == null) {
+            LOG.error("Device requested data with invalid id: {}", dataId);
+            return GdiDataTransferService.DataTransferService.DataDownloadResponse.newBuilder()
+                    .setStatus(GdiDataTransferService.DataTransferService.Status.INVALID_ID)
+                    .setId(dataId)
+                    .setOffset(offset)
+                    .build();
+        }
+        final int maxChunkSize = dataDownloadRequest.hasMaxChunkSize() ? dataDownloadRequest.getMaxChunkSize() : Integer.MAX_VALUE;
+        final byte[] chunk = data.getDataChunk(offset, maxChunkSize);
+        if (chunk == null) {
+            LOG.error("Device requested data with invalid offset: {}", offset);
+            return GdiDataTransferService.DataTransferService.DataDownloadResponse.newBuilder()
+                    .setStatus(GdiDataTransferService.DataTransferService.Status.INVALID_OFFSET)
+                    .setId(dataId)
+                    .setOffset(offset)
+                    .build();
+        }
+        requestInfoById.put(requestId, new RequestInfo(dataId, chunk.length));
+        return GdiDataTransferService.DataTransferService.DataDownloadResponse.newBuilder()
+                .setStatus(GdiDataTransferService.DataTransferService.Status.SUCCESS)
+                .setId(dataId)
+                .setOffset(offset)
+                .setPayload(ByteString.copyFrom(chunk))
+                .build();
+    }
+
+    public static int registerData(final byte[] data) {
+        int id = idCounter.getAndIncrement();
+        LOG.info("New data will be sent to the device (id: {}, size: {})", id, data.length);
+        dataById.put(id, new Data(data));
+        return id;
+    }
+
+    public static void onDataSuccessfullyReceived(final int requestId) {
+        final RequestInfo requestInfo = requestInfoById.get(requestId);
+        requestInfoById.remove(requestId);
+        if (requestInfo == null) {
+            return;
+        }
+        final Data data = dataById.get(requestInfo.dataId);
+        if (data == null) {
+            return;
+        }
+        int dataLeft = data.onDataSuccessfullyReceived(requestInfo.requestDataLength);
+        if (dataLeft == 0) {
+            LOG.info("Data successfully sent to the device (id: {}, size: {})", requestInfo.dataId, data.data.length);
+            dataById.remove(requestInfo.dataId);
+        } else {
+            LOG.debug("Data chunk successfully sent to the device (dataId: {}, requestId: {}, data left: {})", requestInfo.dataId, requestId, dataLeft);
+        }
+    }
+
+    private static class RequestInfo {
+        private final int dataId;
+        private final int requestDataLength;
+
+        private RequestInfo(int dataId, int requestDataLength) {
+            this.dataId = dataId;
+            this.requestDataLength = requestDataLength;
+        }
+    }
+
+    private static class Data {
+        // TODO Wouldn't it be better to store data as streams?
+        // Because now we have to store the whole data in RAM.
+        private final byte[] data;
+        private final AtomicInteger dataLeft;
+
+        private Data(byte[] data) {
+            this.data = data;
+            this.dataLeft = new AtomicInteger(data.length);
+        }
+
+        private byte[] getDataChunk(final int offset, final int maxChunkSize) {
+            if (offset < 0 || offset >= data.length) {
+                return null;
+            }
+            return Arrays.copyOfRange(data, offset, Math.min(offset + maxChunkSize, data.length));
+        }
+
+        private int onDataSuccessfullyReceived(int chunkSize) {
+            // TODO Does this work properly?
+            // Problems can arise when the app receives two ACKs for the same data.
+            // It can be solved by storing information about what data was ACKed instead of just dataLeft variable.
+            return dataLeft.addAndGet(-chunkSize);
+        }
+    }
+}

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/garmin/http/EphemerisHandler.java

@@ -0,0 +1,40 @@
+package nodomain.freeyourgadget.gadgetbridge.service.devices.garmin.http;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Map;
+
+import nodomain.freeyourgadget.gadgetbridge.service.devices.garmin.GarminSupport;
+
+public class EphemerisHandler {
+    private static final Logger LOG = LoggerFactory.getLogger(EphemerisHandler.class);
+    private final GarminSupport deviceSupport;
+
+    public EphemerisHandler(GarminSupport deviceSupport) {
+        this.deviceSupport = deviceSupport;
+    }
+
+    public byte[] handleEphemerisRequest(final String path, final Map<String, String> query) {
+        try {
+            final File exportDirectory = deviceSupport.getWritableExportDirectory();
+            final File ephemerisDataFile = new File(exportDirectory, "CPE.BIN");
+            if (!ephemerisDataFile.exists() || !ephemerisDataFile.isFile()) {
+                throw new IOException("Cannot locate CPE.BIN file in export/import directory.");
+            }
+            final byte[] bytes = new byte[(int) ephemerisDataFile.length()];
+            final BufferedInputStream bis = new BufferedInputStream(new FileInputStream(ephemerisDataFile));
+            final DataInputStream dis = new DataInputStream(bis);
+            dis.readFully(bytes);
+            return bytes;
+        } catch (IOException e) {
+            LOG.error("Unable to obtain ephemeris data.", e);
+            return null;
+        }
+    }
+}

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/garmin/http/HttpHandler.java

@@ -19,6 +19,7 @@ import java.util.Map;
 import java.util.zip.GZIPOutputStream;
 
 import nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr.GdiHttpService;
+import nodomain.freeyourgadget.gadgetbridge.service.devices.garmin.GarminSupport;
 import nodomain.freeyourgadget.gadgetbridge.util.HttpUtils;
 
 public class HttpHandler {
@@ -28,7 +29,13 @@ public class HttpHandler {
             //.serializeNulls()
             .create();
 
-    public static GdiHttpService.HttpService handle(final GdiHttpService.HttpService httpService) {
+    private final EphemerisHandler ephemerisHandler;
+
+    public HttpHandler(GarminSupport deviceSupport) {
+        ephemerisHandler = new EphemerisHandler(deviceSupport);
+    }
+
+    public GdiHttpService.HttpService handle(final GdiHttpService.HttpService httpService) {
         if (httpService.hasRawRequest()) {
             final GdiHttpService.HttpService.RawResponse rawResponse = handleRawRequest(httpService.getRawRequest());
             if (rawResponse != null) {
@@ -44,7 +51,7 @@ public class HttpHandler {
         return null;
     }
 
-    public static GdiHttpService.HttpService.RawResponse handleRawRequest(final GdiHttpService.HttpService.RawRequest rawRequest) {
+    public GdiHttpService.HttpService.RawResponse handleRawRequest(final GdiHttpService.HttpService.RawRequest rawRequest) {
         final String urlString = rawRequest.getUrl();
         LOG.debug("Got rawRequest: {} - {}", rawRequest.getMethod(), urlString);
 
@@ -58,21 +65,53 @@ public class HttpHandler {
 
         final String path = url.getPath();
         final Map<String, String> query = HttpUtils.urlQueryParameters(url);
-        final Map<String, String> requestHeaders = headersToMap(rawRequest.getHeaderList());
 
-        final byte[] responseBody;
-        final List<GdiHttpService.HttpService.Header> responseHeaders = new ArrayList<>();
         if (path.startsWith("/weather/")) {
-            LOG.debug("Got weather request for {}", path);
+            LOG.info("Got weather request for {}", path);
-            final Object obj = WeatherHandler.handleWeatherRequest(path, query);
+            final Object weatherData = WeatherHandler.handleWeatherRequest(path, query);
-            if (obj == null) {
+            if (weatherData == null) {
                 return null;
             }
-            final String json = GSON.toJson(obj);
+            final String json = GSON.toJson(weatherData);
             LOG.debug("Weather response: {}", json);
+            return createRawResponse(rawRequest, json.getBytes(StandardCharsets.UTF_8), "application/json");
+        } else if (path.startsWith("/ephemeris/")) {
+            LOG.info("Got ephemeris request for {}", path);
+            byte[] ephemerisData = ephemerisHandler.handleEphemerisRequest(path, query);
+            if (ephemerisData == null) {
+                return null;
+            }
+            LOG.debug("Successfully obtained ephemeris data (length: {})", ephemerisData.length);
+            return createRawResponse(rawRequest, ephemerisData, "application/x-tar");
+        } else {
+            LOG.warn("Unhandled path {}", urlString);
+            return null;
+        }
+    }
 
-            final byte[] stringBytes = json.getBytes(StandardCharsets.UTF_8);
+    private static GdiHttpService.HttpService.RawResponse createRawResponse(
+            final GdiHttpService.HttpService.RawRequest rawRequest,
+            final byte[] data,
+            final String contentType
+    ) {
+        if (rawRequest.hasUseDataXfer() && rawRequest.getUseDataXfer()) {
+            LOG.debug("Data will be returned using data_xfer");
+            int id = DataTransferHandler.registerData(data);
+            return GdiHttpService.HttpService.RawResponse.newBuilder()
+                    .setStatus(GdiHttpService.HttpService.Status.OK)
+                    .setHttpStatus(200)
+                    .setXferData(
+                            GdiHttpService.HttpService.DataTransferItem.newBuilder()
+                                    .setId(id)
+                                    .setSize(data.length)
+                                    .build()
+                    )
+                    .build();
+        }
 
+        final Map<String, String> requestHeaders = headersToMap(rawRequest.getHeaderList());
+        final List<GdiHttpService.HttpService.Header> responseHeaders = new ArrayList<>();
+        final byte[] responseBody;
         if ("gzip".equals(requestHeaders.get("accept-encoding"))) {
             responseHeaders.add(
                     GdiHttpService.HttpService.Header.newBuilder()
@@ -83,7 +122,7 @@ public class HttpHandler {
 
             final ByteArrayOutputStream baos = new ByteArrayOutputStream();
             try (GZIPOutputStream gzos = new GZIPOutputStream(baos)) {
-                    gzos.write(stringBytes);
+                gzos.write(data);
                 gzos.finish();
                 gzos.flush();
                 responseBody = baos.toByteArray();
@@ -92,19 +131,15 @@ public class HttpHandler {
                 return null;
             }
         } else {
-                responseBody = stringBytes;
+            responseBody = data;
         }
 
         responseHeaders.add(
                 GdiHttpService.HttpService.Header.newBuilder()
                         .setKey("Content-Type")
-                            .setValue("application/json")
+                        .setValue(contentType)
                         .build()
         );
-        } else {
-            LOG.warn("Unhandled path {}", urlString);
-            return null;
-        }
 
         return GdiHttpService.HttpService.RawResponse.newBuilder()
                 .setStatus(GdiHttpService.HttpService.Status.OK)

app/src/main/proto/garmin_vivomovehr/gdi_data_transfer_service.proto

@@ -0,0 +1,30 @@
+syntax = "proto2";
+
+package garmin_vivomovehr;
+
+option java_package = "nodomain.freeyourgadget.gadgetbridge.proto.vivomovehr";
+
+message DataTransferService {
+  enum Status {
+    UNKNOWN = 0;
+    SUCCESS = 1;
+    INVALID_ID = 2;
+    INVALID_OFFSET = 3;
+  }
+
+  optional DataDownloadRequest dataDownloadRequest = 1;
+  optional DataDownloadResponse dataDownloadResponse = 2;
+
+  message DataDownloadRequest {
+    required uint32 id = 1;
+    required uint32 offset = 2;
+    optional uint32 maxChunkSize = 3;
+  }
+
+  message DataDownloadResponse {
+    required Status status = 1;
+    required uint32 id = 2;
+    required uint32 offset = 3;
+    optional bytes payload = 4;
+  }
+}

app/src/main/proto/garmin_vivomovehr/gdi_http_service.proto

@@ -30,15 +30,22 @@ message HttpService {
     required string url = 1;
     optional Method method = 3;
     repeated Header header = 5;
+    optional bool useDataXfer = 6;
   }
 
   message RawResponse {
     optional Status status = 1;
     optional uint32 httpStatus = 2;
     optional bytes body = 3;
+    optional DataTransferItem xferData = 4;
     repeated Header header = 5;
   }
 
+  message DataTransferItem {
+    required uint32 id = 1;
+    required uint32 size = 2;
+  }
+
   message Header {
     required string key = 1;
     required string value = 2;

app/src/main/proto/garmin_vivomovehr/gdi_smart_proto.proto

@@ -8,6 +8,7 @@ import "garmin_vivomovehr/gdi_device_status.proto";
 import "garmin_vivomovehr/gdi_find_my_watch.proto";
 import "garmin_vivomovehr/gdi_core.proto";
 import "garmin_vivomovehr/gdi_http_service.proto";
+import "garmin_vivomovehr/gdi_data_transfer_service.proto";
 import "garmin_vivomovehr/gdi_sms_notification.proto";
 import "garmin_vivomovehr/gdi_calendar_service.proto";
 import "garmin_vivomovehr/gdi_settings_service.proto";
@@ -15,6 +16,7 @@ import "garmin_vivomovehr/gdi_settings_service.proto";
 message Smart {
     optional CalendarService calendar_service = 1;
     optional HttpService http_service = 2;
+    optional DataTransferService data_transfer_service = 7;
     optional DeviceStatusService device_status_service = 8;
     optional FindMyWatchService find_my_watch_service = 12;
     optional CoreService core_service = 13;