Add support for marketplace transformations (#4216)

* Add support for marketplace transformations

Signed-off-by: Jan N. Klug <github@klug.nrw>
This commit is contained in:
J-N-K
2024-05-26 16:06:44 +02:00
committed by GitHub
parent ad74d5a296
commit 4b4f25d31b
10 changed files with 266 additions and 49 deletions
@@ -12,7 +12,8 @@
*/
package org.openhab.core.addon.marketplace.karaf.internal.community;
import static org.openhab.core.addon.marketplace.internal.community.CommunityMarketplaceAddonService.KAR_CONTENT_TYPE;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.KAR_CONTENT_TYPE;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.KAR_DOWNLOAD_URL_PROPERTY;
import java.io.File;
import java.io.IOException;
@@ -55,7 +56,6 @@ public class CommunityKarafAddonHandler implements MarketplaceAddonHandler {
private static final Path KAR_CACHE_PATH = Path.of(OpenHAB.getUserDataFolder(), "marketplace", "kar");
private static final List<String> SUPPORTED_EXT_TYPES = List.of("automation", "binding", "misc", "persistence",
"transformation", "ui", "voice");
private static final String KAR_DOWNLOAD_URL_PROPERTY = "kar_download_url";
private static final String KAR_EXTENSION = ".kar";
private final Logger logger = LoggerFactory.getLogger(CommunityKarafAddonHandler.class);
@@ -31,6 +31,12 @@
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.openhab.core.bundles</groupId>
<artifactId>org.openhab.core.transform</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.openhab.core.bundles</groupId>
<artifactId>org.openhab.core.config.core</artifactId>
@@ -0,0 +1,31 @@
/**
* Copyright (c) 2010-2024 Contributors to the openHAB project
*
* See the NOTICE file(s) distributed with this work for additional
* information.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.openhab.core.addon.marketplace;
/**
* This class contains constants used in marketplace add-on services
*
* @author Jan N. Klug - Initial contribution
*/
public class MarketplaceConstants {
public static final String JAR_CONTENT_TYPE = "application/vnd.openhab.bundle";
public static final String KAR_CONTENT_TYPE = "application/vnd.openhab.feature;type=karfile";
public static final String RULETEMPLATES_CONTENT_TYPE = "application/vnd.openhab.ruletemplate";
public static final String UIWIDGETS_CONTENT_TYPE = "application/vnd.openhab.uicomponent;type=widget";
public static final String BLOCKLIBRARIES_CONTENT_TYPE = "application/vnd.openhab.uicomponent;type=blocks";
public static final String TRANSFORMATIONS_CONTENT_TYPE = "application/vnd.openhab.transformation";
public static final String JAR_DOWNLOAD_URL_PROPERTY = "jar_download_url";
public static final String KAR_DOWNLOAD_URL_PROPERTY = "kar_download_url";
public static final String JSON_DOWNLOAD_URL_PROPERTY = "json_download_url";
public static final String YAML_DOWNLOAD_URL_PROPERTY = "yaml_download_url";
}
@@ -12,6 +12,10 @@
*/
package org.openhab.core.addon.marketplace.internal.community;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.BLOCKLIBRARIES_CONTENT_TYPE;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.YAML_DOWNLOAD_URL_PROPERTY;
import static org.openhab.core.addon.marketplace.internal.community.CommunityMarketplaceAddonService.YAML_CONTENT_PROPERTY;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
@@ -45,10 +49,6 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
@Component(immediate = true)
@NonNullByDefault
public class CommunityBlockLibaryAddonHandler implements MarketplaceAddonHandler {
private static final String YAML_DOWNLOAD_URL_PROPERTY = "yaml_download_url";
private static final String YAML_CONTENT_PROPERTY = "yaml_content";
private static final String BLOCKLIBRARIES_CONTENT_TYPE = "application/vnd.openhab.uicomponent;type=blocks";
private final Logger logger = LoggerFactory.getLogger(CommunityBlockLibaryAddonHandler.class);
ObjectMapper yamlMapper;
@@ -12,7 +12,8 @@
*/
package org.openhab.core.addon.marketplace.internal.community;
import static org.openhab.core.addon.marketplace.internal.community.CommunityMarketplaceAddonService.JAR_CONTENT_TYPE;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.JAR_CONTENT_TYPE;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.JAR_DOWNLOAD_URL_PROPERTY;
import java.net.MalformedURLException;
import java.net.URL;
@@ -44,7 +45,6 @@ import org.osgi.service.component.annotations.Component;
public class CommunityBundleAddonHandler extends MarketplaceBundleInstaller implements MarketplaceAddonHandler {
private static final List<String> SUPPORTED_EXT_TYPES = List.of("automation", "binding", "misc", "persistence",
"transformation", "ui", "voice");
private static final String JAR_DOWNLOAD_URL_PROPERTY = "jar_download_url";
private final ScheduledExecutorService scheduler = ThreadPoolManager
.getScheduledPool(ThreadPoolManager.THREAD_POOL_NAME_COMMON);
@@ -13,6 +13,7 @@
package org.openhab.core.addon.marketplace.internal.community;
import static org.openhab.core.addon.Addon.CODE_MATURITY_LEVELS;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.*;
import java.io.InputStreamReader;
import java.io.Reader;
@@ -73,11 +74,9 @@ import org.slf4j.LoggerFactory;
@ConfigurableService(category = "system", label = CommunityMarketplaceAddonService.SERVICE_NAME, description_uri = CommunityMarketplaceAddonService.CONFIG_URI)
@NonNullByDefault
public class CommunityMarketplaceAddonService extends AbstractRemoteAddonService {
public static final String JAR_CONTENT_TYPE = "application/vnd.openhab.bundle";
public static final String KAR_CONTENT_TYPE = "application/vnd.openhab.feature;type=karfile";
public static final String RULETEMPLATES_CONTENT_TYPE = "application/vnd.openhab.ruletemplate";
public static final String UIWIDGETS_CONTENT_TYPE = "application/vnd.openhab.uicomponent;type=widget";
public static final String BLOCKLIBRARIES_CONTENT_TYPE = "application/vnd.openhab.uicomponent;type=blocks";
public static final String CODE_CONTENT_SUFFIX = "_content";
public static final String JSON_CONTENT_PROPERTY = "json" + CODE_CONTENT_SUFFIX;
public static final String YAML_CONTENT_PROPERTY = "yaml" + CODE_CONTENT_SUFFIX;
// constants for the configuration properties
static final String SERVICE_NAME = "Community Marketplace";
@@ -95,14 +94,15 @@ public class CommunityMarketplaceAddonService extends AbstractRemoteAddonService
private static final String SERVICE_ID = "marketplace";
private static final String ADDON_ID_PREFIX = SERVICE_ID + ":";
private static final String JSON_CODE_MARKUP_START = "<pre><code class=\"lang-json\">";
private static final String YAML_CODE_MARKUP_START = "<pre><code class=\"lang-yaml\">";
private static final String CODE_MARKUP_END = "</code></pre>";
private static final Pattern CODE_MARKUP_PATTERN = Pattern.compile(
"<pre(?: data-code-wrap=\"[a-z]+\")?><code class=\"lang-(?<lang>[a-z]+)\">(?<content>.*?)</code></pre>",
Pattern.DOTALL);
private static final Integer BUNDLES_CATEGORY = 73;
private static final Integer RULETEMPLATES_CATEGORY = 74;
private static final Integer UIWIDGETS_CATEGORY = 75;
private static final Integer BLOCKLIBRARIES_CATEGORY = 76;
private static final Integer TRANSFORMATIONS_CATEGORY = 80;
private static final String PUBLISHED_TAG = "published";
@@ -241,7 +241,9 @@ public class CommunityMarketplaceAddonService extends AbstractRemoteAddonService
private @Nullable AddonType getAddonType(@Nullable Integer category, List<String> tags) {
// check if we can determine the addon type from the category
if (RULETEMPLATES_CATEGORY.equals(category)) {
if (TRANSFORMATIONS_CATEGORY.equals(category)) {
return AddonType.TRANSFORMATION;
} else if (RULETEMPLATES_CATEGORY.equals(category)) {
return AddonType.AUTOMATION;
} else if (UIWIDGETS_CATEGORY.equals(category)) {
return AddonType.UI;
@@ -259,7 +261,9 @@ public class CommunityMarketplaceAddonService extends AbstractRemoteAddonService
private String getContentType(@Nullable Integer category, List<String> tags) {
// check if we can determine the addon type from the category
if (RULETEMPLATES_CATEGORY.equals(category)) {
if (TRANSFORMATIONS_CATEGORY.equals(category)) {
return TRANSFORMATIONS_CONTENT_TYPE;
} else if (RULETEMPLATES_CATEGORY.equals(category)) {
return RULETEMPLATES_CONTENT_TYPE;
} else if (UIWIDGETS_CATEGORY.equals(category)) {
return UIWIDGETS_CONTENT_TYPE;
@@ -403,18 +407,18 @@ public class CommunityMarketplaceAddonService extends AbstractRemoteAddonService
if (topic.postStream.posts[0].linkCounts != null) {
for (DiscoursePostLink postLink : topic.postStream.posts[0].linkCounts) {
if (postLink.url.endsWith(".jar")) {
properties.put("jar_download_url", postLink.url);
properties.put(JAR_DOWNLOAD_URL_PROPERTY, postLink.url);
id = determineIdFromUrl(postLink.url);
}
if (postLink.url.endsWith(".kar")) {
properties.put("kar_download_url", postLink.url);
properties.put(KAR_DOWNLOAD_URL_PROPERTY, postLink.url);
id = determineIdFromUrl(postLink.url);
}
if (postLink.url.endsWith(".json")) {
properties.put("json_download_url", postLink.url);
properties.put(JSON_DOWNLOAD_URL_PROPERTY, postLink.url);
}
if (postLink.url.endsWith(".yaml")) {
properties.put("yaml_download_url", postLink.url);
properties.put(YAML_DOWNLOAD_URL_PROPERTY, postLink.url);
}
}
}
@@ -423,17 +427,10 @@ public class CommunityMarketplaceAddonService extends AbstractRemoteAddonService
id = topic.id.toString(); // this is a fallback if we couldn't find a better id
}
if (detailedDescription.contains(JSON_CODE_MARKUP_START)) {
String jsonContent = detailedDescription.substring(
detailedDescription.indexOf(JSON_CODE_MARKUP_START) + JSON_CODE_MARKUP_START.length(),
detailedDescription.indexOf(CODE_MARKUP_END, detailedDescription.indexOf(JSON_CODE_MARKUP_START)));
properties.put("json_content", unescapeEntities(jsonContent));
}
if (detailedDescription.contains(YAML_CODE_MARKUP_START)) {
String yamlContent = detailedDescription.substring(
detailedDescription.indexOf(YAML_CODE_MARKUP_START) + YAML_CODE_MARKUP_START.length(),
detailedDescription.indexOf(CODE_MARKUP_END, detailedDescription.indexOf(YAML_CODE_MARKUP_START)));
properties.put("yaml_content", unescapeEntities(yamlContent));
Matcher codeMarkup = CODE_MARKUP_PATTERN.matcher(detailedDescription);
if (codeMarkup.find()) {
properties.put(codeMarkup.group("lang") + CODE_CONTENT_SUFFIX,
unescapeEntities(codeMarkup.group("content")));
}
// try to use a handler to determine if the add-on is installed
@@ -12,6 +12,10 @@
*/
package org.openhab.core.addon.marketplace.internal.community;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.*;
import static org.openhab.core.addon.marketplace.internal.community.CommunityMarketplaceAddonService.JSON_CONTENT_PROPERTY;
import static org.openhab.core.addon.marketplace.internal.community.CommunityMarketplaceAddonService.YAML_CONTENT_PROPERTY;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
@@ -42,12 +46,6 @@ import org.slf4j.LoggerFactory;
@Component(immediate = true)
@NonNullByDefault
public class CommunityRuleTemplateAddonHandler implements MarketplaceAddonHandler {
private static final String JSON_DOWNLOAD_URL_PROPERTY = "json_download_url";
private static final String YAML_DOWNLOAD_URL_PROPERTY = "yaml_download_url";
private static final String JSON_CONTENT_PROPERTY = "json_content";
private static final String YAML_CONTENT_PROPERTY = "yaml_content";
private static final String RULETEMPLATES_CONTENT_TYPE = "application/vnd.openhab.ruletemplate";
private final Logger logger = LoggerFactory.getLogger(CommunityRuleTemplateAddonHandler.class);
private final MarketplaceRuleTemplateProvider marketplaceRuleTemplateProvider;
@@ -0,0 +1,183 @@
/**
* Copyright (c) 2010-2024 Contributors to the openHAB project
*
* See the NOTICE file(s) distributed with this work for additional
* information.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.openhab.core.addon.marketplace.internal.community;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.*;
import static org.openhab.core.addon.marketplace.internal.community.CommunityMarketplaceAddonService.JSON_CONTENT_PROPERTY;
import static org.openhab.core.addon.marketplace.internal.community.CommunityMarketplaceAddonService.YAML_CONTENT_PROPERTY;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.concurrent.CopyOnWriteArrayList;
import org.eclipse.jdt.annotation.NonNullByDefault;
import org.openhab.core.addon.Addon;
import org.openhab.core.addon.marketplace.MarketplaceAddonHandler;
import org.openhab.core.addon.marketplace.MarketplaceHandlerException;
import org.openhab.core.common.registry.ProviderChangeListener;
import org.openhab.core.storage.Storage;
import org.openhab.core.storage.StorageService;
import org.openhab.core.transform.ManagedTransformationProvider.PersistedTransformation;
import org.openhab.core.transform.Transformation;
import org.openhab.core.transform.TransformationProvider;
import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.introspect.AnnotationIntrospectorPair;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.google.gson.Gson;
import com.google.gson.JsonParseException;
/**
* A {@link MarketplaceAddonHandler} implementation, which handles community provided transformations
*
* @author Jan N. Klugg - Initial contribution
*
*/
@Component(immediate = true)
@NonNullByDefault
public class CommunityTransformationAddonHandler implements MarketplaceAddonHandler, TransformationProvider {
private final Logger logger = LoggerFactory.getLogger(CommunityTransformationAddonHandler.class);
private final ObjectMapper yamlMapper;
private final Gson gson = new Gson();
private final Storage<PersistedTransformation> storage;
private final List<ProviderChangeListener<Transformation>> changeListeners = new CopyOnWriteArrayList<>();
@Activate
public CommunityTransformationAddonHandler(final @Reference StorageService storageService) {
this.storage = storageService.getStorage("org.openhab.marketplace.transformation");
this.yamlMapper = new ObjectMapper(new YAMLFactory());
yamlMapper.findAndRegisterModules();
this.yamlMapper.setDateFormat(new SimpleDateFormat("MMM d, yyyy, hh:mm:ss aa", Locale.ENGLISH));
yamlMapper.setAnnotationIntrospector(new AnnotationIntrospectorPair(new SerializedNameAnnotationIntrospector(),
yamlMapper.getSerializationConfig().getAnnotationIntrospector()));
}
@Override
public boolean supports(String type, String contentType) {
return "transformation".equals(type) && TRANSFORMATIONS_CONTENT_TYPE.equals(contentType);
}
@Override
public boolean isInstalled(String id) {
return storage.containsKey(id);
}
@Override
public void install(Addon addon) throws MarketplaceHandlerException {
try {
String yamlDownloadUrl = (String) addon.getProperties().get(YAML_DOWNLOAD_URL_PROPERTY);
String yamlContent = (String) addon.getProperties().get(YAML_CONTENT_PROPERTY);
String jsonDownloadUrl = (String) addon.getProperties().get(JSON_DOWNLOAD_URL_PROPERTY);
String jsonContent = (String) addon.getProperties().get(JSON_CONTENT_PROPERTY);
PersistedTransformation persistedTransformation;
if (yamlDownloadUrl != null) {
persistedTransformation = addTransformationFromYAML(addon.getUid(),
downloadTransformation(yamlDownloadUrl));
} else if (yamlContent != null) {
persistedTransformation = addTransformationFromYAML(addon.getUid(), yamlContent);
} else if (jsonDownloadUrl != null) {
persistedTransformation = addTransformationFromJSON(addon.getUid(),
downloadTransformation(jsonDownloadUrl));
} else if (jsonContent != null) {
persistedTransformation = addTransformationFromJSON(addon.getUid(), jsonContent);
} else {
throw new IllegalArgumentException(
"Couldn't find the transformation in the add-on entry. The starting code fence may not be marked as ```yaml");
}
Transformation transformation = map(persistedTransformation);
changeListeners.forEach(l -> l.added(this, transformation));
} catch (IOException e) {
logger.error("Transformation from marketplace cannot be downloaded: {}", e.getMessage());
throw new MarketplaceHandlerException("Transformation cannot be downloaded.", e);
} catch (Exception e) {
logger.error("Transformation from marketplace is invalid: {}", e.getMessage());
throw new MarketplaceHandlerException("Transformation is not valid.", e);
}
}
@Override
public void uninstall(Addon addon) throws MarketplaceHandlerException {
PersistedTransformation toRemoveElement = storage.remove(addon.getUid());
if (toRemoveElement != null) {
Transformation toRemove = map(toRemoveElement);
changeListeners.forEach(l -> l.removed(this, toRemove));
}
}
private String downloadTransformation(String urlString) throws IOException {
URL u = new URL(urlString);
try (InputStream in = u.openStream()) {
return new String(in.readAllBytes(), StandardCharsets.UTF_8);
}
}
private PersistedTransformation addTransformationFromYAML(String id, String yaml) {
try {
PersistedTransformation transformation = yamlMapper.readValue(yaml, PersistedTransformation.class);
storage.put(id, transformation);
return transformation;
} catch (IOException e) {
logger.error("Unable to parse YAML: {}", e.getMessage());
throw new IllegalArgumentException("Unable to parse YAML");
}
}
private PersistedTransformation addTransformationFromJSON(String id, String json) {
try {
PersistedTransformation transformation = gson.fromJson(json, PersistedTransformation.class);
storage.put(id, transformation);
return transformation;
} catch (JsonParseException e) {
logger.error("Unable to parse JSON: {}", e.getMessage());
throw new IllegalArgumentException("Unable to parse JSON");
}
}
@Override
public void addProviderChangeListener(ProviderChangeListener<Transformation> listener) {
changeListeners.add(listener);
}
@Override
public Collection<Transformation> getAll() {
return storage.getValues().stream().filter(Objects::nonNull).map(Objects::requireNonNull).map(this::map)
.toList();
}
@Override
public void removeProviderChangeListener(ProviderChangeListener<Transformation> listener) {
changeListeners.remove(listener);
}
private Transformation map(PersistedTransformation persistedTransformation) {
return new Transformation(persistedTransformation.uid, persistedTransformation.label,
persistedTransformation.type, persistedTransformation.configuration);
}
}
@@ -12,6 +12,10 @@
*/
package org.openhab.core.addon.marketplace.internal.community;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.UIWIDGETS_CONTENT_TYPE;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.YAML_DOWNLOAD_URL_PROPERTY;
import static org.openhab.core.addon.marketplace.internal.community.CommunityMarketplaceAddonService.YAML_CONTENT_PROPERTY;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
@@ -46,14 +50,10 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
@Component(immediate = true)
@NonNullByDefault
public class CommunityUIWidgetAddonHandler implements MarketplaceAddonHandler {
private static final String YAML_DOWNLOAD_URL_PROPERTY = "yaml_download_url";
private static final String YAML_CONTENT_PROPERTY = "yaml_content";
private static final String UIWIDGETS_CONTENT_TYPE = "application/vnd.openhab.uicomponent;type=widget";
private final Logger logger = LoggerFactory.getLogger(CommunityUIWidgetAddonHandler.class);
ObjectMapper yamlMapper;
private UIComponentRegistry widgetRegistry;
private final ObjectMapper yamlMapper;
private final UIComponentRegistry widgetRegistry;
@Activate
public CommunityUIWidgetAddonHandler(final @Reference UIComponentRegistryFactory uiComponentRegistryFactory) {
@@ -12,6 +12,8 @@
*/
package org.openhab.core.addon.marketplace.internal.json;
import static org.openhab.core.addon.marketplace.MarketplaceConstants.*;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
@@ -172,13 +174,13 @@ public class JsonAddonService extends AbstractRemoteAddonService {
Map<String, Object> properties = new HashMap<>();
if (addonEntry.url.endsWith(".jar")) {
properties.put("jar_download_url", addonEntry.url);
properties.put(JAR_DOWNLOAD_URL_PROPERTY, addonEntry.url);
} else if (addonEntry.url.endsWith(".kar")) {
properties.put("kar_download_url", addonEntry.url);
properties.put(KAR_DOWNLOAD_URL_PROPERTY, addonEntry.url);
} else if (addonEntry.url.endsWith(".json")) {
properties.put("json_download_url", addonEntry.url);
properties.put(JSON_DOWNLOAD_URL_PROPERTY, addonEntry.url);
} else if (addonEntry.url.endsWith(".yaml")) {
properties.put("yaml_download_url", addonEntry.url);
properties.put(YAML_DOWNLOAD_URL_PROPERTY, addonEntry.url);
}
boolean compatible = true;