diff --git a/CODEOWNERS b/CODEOWNERS index 0e80ca1e07..33b01fe933 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -473,6 +473,7 @@ /bundles/org.openhab.io.metrics/ @pravussum /bundles/org.openhab.io.neeo/ @morph166955 /bundles/org.openhab.io.openhabcloud/ @kaikreuzer +/bundles/org.openhab.io.yamlcomposer/ @jimtng /bundles/org.openhab.persistence.dynamodb/ @ssalonen /bundles/org.openhab.persistence.influxdb/ @lujop /bundles/org.openhab.persistence.inmemory/ @J-N-K diff --git a/bundles/org.openhab.io.yamlcomposer/NOTICE b/bundles/org.openhab.io.yamlcomposer/NOTICE new file mode 100644 index 0000000000..38d625e349 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/NOTICE @@ -0,0 +1,13 @@ +This content is produced and maintained by the openHAB project. + +* Project home: https://www.openhab.org + +== Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License 2.0 which is available at +https://www.eclipse.org/legal/epl-2.0/. + +== Source Code + +https://github.com/openhab/openhab-addons diff --git a/bundles/org.openhab.io.yamlcomposer/README.md b/bundles/org.openhab.io.yamlcomposer/README.md new file mode 100644 index 0000000000..ff075c030f --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/README.md @@ -0,0 +1,329 @@ +# YAML Composer + +YAML Composer introduces extended YAML features that make openHAB configuration more modular, reusable, and maintainable. These features let you structure configuration as composable building blocks rather than large, repetitive files. + +The add-on loads enhanced-syntax YAML files from `OPENHAB_CONF/yamlcomposer/` and compiles them into fully resolved plain YAML written to `OPENHAB_CONF/yaml/composed/`. + +[[toc]] + +## Feature Summary + +YAML Composer adds several enhancements on top of standard YAML. +Each feature addresses a different kind of reuse, composition, or abstraction to help you build cleaner and more maintainable YAML. + +| Feature | Purpose | Typical Use | +|--------------------------------------------|-----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------| +| **Variables and Substitution (`${..}`)** | Insert dynamic values or evaluate expressions | Build labels, topics, IDs, or computed values | +| **Conditionals (`!if`)** | Conditionally include or exclude YAML blocks | Enable or disable features when using packages or template flags | +| **Include (`!include`)** | Insert the contents of another file | Reuse YAML across files; parameterize reusable blocks | +| **Templates (`!insert`)** | Reuse YAML defined within the same file | Local parameterized blocks; reusable channel or item fragments | +| **Packages** | Bundle multiple top-level sections into one reusable unit | Define reusable device structures containing things, items, metadata; sourced from external files or templates | +| **Anchors and Aliases (`&name`, `*name`)** | Define small, reusable YAML fragments | Static defaults, shared fields | +| **Merge Keys (`<<:`)** | Combine mappings from multiple sources | Layer defaults, override fields, compose structures | + +Each feature has a dedicated documentation page: + +- [Variables and Substitution](doc/variables.md) +- [Conditionals](doc/conditionals.md) +- [Include](doc/include.md) +- [Templates](doc/templates.md) +- [Packages](doc/packages.md) +- [Anchors and Aliases](doc/anchors.md) +- [Merge Keys](doc/merge-keys.md) + +These features can be used independently, but they become especially powerful when combined. + +For a general introduction to YAML, see [YAML Basics](doc/basics.md). + +## Packaging Example + +**CONF/yamlcomposer/LivingRoom.yaml:** + +```yaml +version: 1 + +variables: + # Captures "LivingRoom" from the filename automatically + location: ${__FILE_NAME__} # => LivingRoom + +packages: + Light1: !include + file: $pkg/zigbee_light.inc.yaml + vars: &LIGHT_VARS + color_temperature: {} # Include color temperature feature + power: # Customize the power item + groups: + - gInsideLights + - gSmartLights + + Light2: !include + file: $pkg/zigbee_light.inc.yaml + vars: + <<: *LIGHT_VARS +``` + +**CONF/yamlcomposer/pkg/zigbee_light.inc.yaml:** + +```yaml +# This is the package file, i.e. the template for a zigbee light +variables: + id: ${location}_${package_id} + thingid: ${id | lower | replace("_", "-")} + equipment: ${id}_Equipment + label: ${id | label} + + # Set defaults + power: &DEFAULTS + groups: [] + dimmer: + <<: *DEFAULTS + +things: + mqtt:topic:${thingid}: + bridge: mqtt:broker:mosquitto + channels: + power: + type: switch + config: + stateTopic: zigbee2mqtt/${thingid}/state + commandTopic: zigbee2mqtt/${thingid}/set/state + + dimmer: + type: dimmer + config: + stateTopic: zigbee2mqtt/${thingid}/brightness + commandTopic: zigbee2mqtt/${thingid}/set/brightness + + <<: !if + if: VARS.containsKey('color_temperature') + then: + color-temperature: !sub + type: dimmer + config: + stateTopic: zigbee2mqtt/${thingid}/color_temp + commandTopic: zigbee2mqtt/${thingid}/set/color_temp + +items: + ${equipment}: + type: Group + label: ${label} Equipment + tags: [Lightbulb] + groups: ${[location]} + + ${id}: + type: Switch + label: ${label} + tags: [Control, Light] + groups: ${[equipment] + power.groups} + channel: mqtt:topic:${thingid}:power + + ${id}_Dimmer: + type: Dimmer + label: ${label} Brightness + tags: [Control, Level] + groups: ${[equipment] + dimmer.groups} + channel: mqtt:topic:${thingid}:dimmer + + <<: !if + if: VARS.containsKey('color_temperature') + then: + ${id}_CT: + type: Dimmer + label: ${label} Color Temperature + tags: [Control, ColorTemperature] + groups: ${[equipment] + color_temperature.groups} + channel: mqtt:topic:${thingid}:color-temperature +``` + +
+Output in CONF/yaml/composed/LivingRoom.yaml: + +```yaml +version: 1 + +things: + mqtt:topic:livingroom-light1: + bridge: mqtt:broker:mosquitto + channels: + power: + type: switch + config: + stateTopic: zigbee2mqtt/livingroom-light1/state + commandTopic: zigbee2mqtt/livingroom-light1/set/state + dimmer: + type: dimmer + config: + stateTopic: zigbee2mqtt/livingroom-light1/brightness + commandTopic: zigbee2mqtt/livingroom-light1/set/brightness + color-temperature: + type: dimmer + config: + stateTopic: zigbee2mqtt/livingroom-light1/color_temp + commandTopic: zigbee2mqtt/livingroom-light1/set/color_temp + + mqtt:topic:livingroom-light2: + bridge: mqtt:broker:mosquitto + channels: + power: + type: switch + config: + stateTopic: zigbee2mqtt/livingroom-light2/state + commandTopic: zigbee2mqtt/livingroom-light2/set/state + dimmer: + type: dimmer + config: + stateTopic: zigbee2mqtt/livingroom-light2/brightness + commandTopic: zigbee2mqtt/livingroom-light2/set/brightness + color-temperature: + type: dimmer + config: + stateTopic: zigbee2mqtt/livingroom-light2/color_temp + commandTopic: zigbee2mqtt/livingroom-light2/set/color_temp + +items: + LivingRoom_Light1_Equipment: + type: Group + label: Living Room Light 1 Equipment + tags: + - Lightbulb + groups: + - LivingRoom + + LivingRoom_Light1: + type: Switch + label: Living Room Light 1 + tags: + - Control + - Light + groups: + - LivingRoom_Light1_Equipment + - gInsideLights + - gSmartLights + channel: mqtt:topic:livingroom-light1:power + + LivingRoom_Light1_Dimmer: + type: Dimmer + label: Living Room Light 1 Brightness + tags: + - Control + - Level + groups: + - LivingRoom_Light1_Equipment + channel: mqtt:topic:livingroom-light1:dimmer + + LivingRoom_Light1_CT: + type: Dimmer + label: Living Room Light 1 Color Temperature + tags: + - Control + - ColorTemperature + groups: + - LivingRoom_Light1_Equipment + channel: mqtt:topic:livingroom-light1:color-temperature + + LivingRoom_Light2_Equipment: + type: Group + label: Living Room Light 2 Equipment + tags: + - Lightbulb + groups: + - LivingRoom + + LivingRoom_Light2: + type: Switch + label: Living Room Light 2 + tags: + - Control + - Light + groups: + - LivingRoom_Light2_Equipment + - gInsideLights + - gSmartLights + channel: mqtt:topic:livingroom-light2:power + + LivingRoom_Light2_Dimmer: + type: Dimmer + label: Living Room Light 2 Brightness + tags: + - Control + - Level + groups: + - LivingRoom_Light2_Equipment + channel: mqtt:topic:livingroom-light2:dimmer + + LivingRoom_Light2_CT: + type: Dimmer + label: Living Room Light 2 Color Temperature + tags: + - Control + - ColorTemperature + groups: + - LivingRoom_Light2_Equipment + channel: mqtt:topic:livingroom-light2:color-temperature +``` + +
+ +## Processing Overview + +YAML Composer reads enhanced-syntax YAML files and performs a compilation pass that expands all extended features into a single, fully resolved YAML document that openHAB can load. + +During compilation, YAML Composer performs the following steps: + +1. **YAML Parsing**: The source file is parsed into an internal structure. +1. **Variable Substitution (`${..}`)**: Expressions are evaluated and injected. +1. **Conditionals (`!if`)**: Conditional logic determines which blocks remain. +1. **Template and Include Expansion**: `!insert` and `!include` bring in referenced content, often using resolved variables. +1. **Package Expansion**: External or local packages are loaded and expanded into their component sections. +1. **Recursive Merging**: Merge keys and package structures are combined into the main document. +1. **Hidden Key Removal**: Keys beginning with `.` are removed from the final output. + +The resulting YAML contains: + +- all variables resolved +- all conditionals evaluated +- all templates and includes expanded +- all anchors and merges applied +- all packages integrated +- all hidden keys removed + +This final compiled YAML is written to `OPENHAB_CONF/yaml/composed/`, where openHAB loads it as Things, Items, Metadata, and other configuration elements as defined by the Core YAML Configuration structure. + +## Hidden Keys + +Keys beginning with a dot (`.`) are treated as hidden. They: + +- exist only during compilation +- are ideal for storing anchors, templates, or shared structures +- keep visible configuration clean +- are removed from the final output + +**Example:** + +```yaml +.base-switch: &BASE_SWITCH + type: Switch + autoupdate: false + +items: + Light1: + <<: *BASE_SWITCH + label: Light One +``` + +## File Structure and Conventions + +YAML files can be organized freely, but the following conventions improve clarity and maintainability: + +- Place `variables:` and `templates:` near the top of the file. +- Group reusable structures under hidden keys. +- Use anchors for static fragments and includes for parameterized ones. +- Keep packages in separate files when they represent reusable device or feature definitions. + +These conventions are optional but help keep complex configurations predictable and easy to navigate. + +### File naming convention + +- Use `*.yaml` or `*.yml` for main source files that produce composed output. +- Use `*.inc.yaml` or `*.inc.yml` for include fragments referenced by `!include`. +- Mixed-role files (both main and include) are not supported. diff --git a/bundles/org.openhab.io.yamlcomposer/doc/anchors.md b/bundles/org.openhab.io.yamlcomposer/doc/anchors.md new file mode 100644 index 0000000000..0bbd93373e --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/doc/anchors.md @@ -0,0 +1,63 @@ +# Anchors & Aliases + +Anchors and aliases provide efficient, in‑file content reuse to reduce repetition without the overhead of external include files or packages. + +[[toc]] + +## Core Concepts + +YAML provides two primary symbols for defining reusable content: + +- **Anchor (`&name`)** — Marks a YAML node (a map, list, or scalar) so it can be reused later. +- **Alias (`*name`)** — References a previously anchored node and injects its content at the current location. + +::: tip Scope Note +Anchors and aliases are **file‑local**. +An anchor defined in an included file cannot be referenced from the main file, and vice versa. +::: + +### Using Hidden Keys for Anchors + +Hidden keys are a convenient place to store anchored nodes without exposing them in the final configuration. + +For a full explanation of hidden keys, see [Hidden Keys](../#hidden-keys). + +Anchors are often combined with the YAML **merge key** (`<<:`), which inserts the contents of an anchored map into the current map. +For full merge‑key semantics, see [Merge Keys](merge-keys.md). + +```yaml +.base-switch: &BASE_SWITCH + type: Switch + autoupdate: false + +items: + Light1: + <<: *BASE_SWITCH + label: Light One +``` + +Anchors can also be applied to scalar values. +This works similarly to using a variable: + +```yaml +.bar: &BAR value + +foo: *BAR +``` + +## When to Use Anchors + +Anchors and aliases are most useful when reusing structural blocks **within the same file**. +They work well for: + +- repeating common fields across many entries +- sharing channel or configuration structures +- keeping small reusable fragments close to where they are used + +Anchors are a YAML‑native mechanism. +They participate in variable substitution, but they do not support cross‑file reuse or template‑style evaluation. + +## Best Practices + +- **Naming** — Use `UPPER_CASE` for anchors to distinguish them from standard keys. +- **Standardization** — Use anchors to enforce consistent structure across repeated YAML blocks within a file. diff --git a/bundles/org.openhab.io.yamlcomposer/doc/basics.md b/bundles/org.openhab.io.yamlcomposer/doc/basics.md new file mode 100644 index 0000000000..122adeff71 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/doc/basics.md @@ -0,0 +1,329 @@ +# Standard YAML Syntax + +This page introduces the standard YAML syntax used throughout this documentation. +It is not a full YAML tutorial. +It provides the foundational concepts needed to understand and work with the examples in the YAML Configuration section. + +The YAML Composer builds on top of standard YAML 1.2 and adds its own processing features such as custom tags and preprocessing behavior. +This page focuses on the core YAML syntax that those features extend. + +For more detailed information on the YAML 1.2 syntax supported in openHAB configurations, see the [YAML 1.2 specification](https://yaml.org/spec/1.2/). + +[[toc]] + +## Indentation and Structure + +YAML uses indentation to represent structure. + +- Indentation defines the nesting level. +- Only spaces are allowed. +- Tabs are not permitted for indentation. +- All items at the same level must use the same indentation. + +openHAB recommends using **two spaces** per indentation level for consistency. + +```yaml +thing: + id: myThing + label: "Example Thing" +``` + +Misaligned indentation is one of the most common YAML errors. + +## Block Style vs Flow Style + +YAML supports two ways of expressing mappings and lists. + +### Block Style (Recommended) + +Block style is the most readable form. + +```yaml +item: + type: Switch + label: "Lamp" + tags: + - Light + - LivingRoom +``` + +### Flow Style + +Flow style is more compact and resembles JSON. +It is functionally identical to block style. + +```yaml +item: { type: Switch, label: "Lamp", tags: [Light, LivingRoom] } +``` + +YAML allows block and flow styles to be mixed freely. + +## Comments + +- Comments begin with `#`. +- Comments continue to the end of the line. +- Comments may appear on their own line or after a value. + +```yaml +# This is a comment describing the thing +thing: + id: myThing # Inline comment + label: "Example Thing" +``` + +## Data Types + +YAML defines three fundamental data types: **scalars**, **lists** (sequences), and **mappings**. + +### Scalars + +Scalars represent simple values such as strings, numbers, booleans, and nulls. + +#### Booleans + +In openHAB YAML files, only the unquoted literals `true`, `True`, `TRUE`, and `false`, `False`, `FALSE` are recognized as boolean values. +`ON`, `OFF`, `Yes`, `No`, `enable`, and `disable` are parsed as plain strings. +To specify `true` or `false` as a string, they must be enclosed in single or double quotes. + +#### Strings + +Strings may be written with or without quotes. + +Unquoted strings are allowed only when the value **cannot** be interpreted as another YAML scalar type such as a number, boolean, or null. +In practice, this means unquoted strings should contain letters or simple punctuation, but **not** values that look like numbers or booleans. + +```yaml +name: KitchenLight +label: Kitchen Light +role: admin +``` + +If a value could be parsed as a number, boolean, or null, it must be quoted to ensure it is treated as a string. + +```yaml +string_number: "123" +string_boolean: "true" +string_null: "null" +``` + +Quotes are required when: + +- The string contains characters YAML may interpret as syntax. +- The string contains leading or trailing spaces. +- The string contains a colon followed by a space, which YAML interprets as a key/value separator. + +```yaml +label: "Kitchen: Main Light" +note: " This string preserves leading spaces" +``` + +Single‑quoted and double‑quoted strings behave differently. + +- Single‑quoted strings treat the content literally. +- Single‑quoted strings do not process escape sequences. +- Single‑quoted strings are useful when you want the text to appear exactly as written. + +```yaml +description: 'This string contains \n and it will not be interpreted as a newline' +``` + +- Double‑quoted strings support escape sequences. +- Double‑quoted strings allow characters such as `\n`, `\t`, and `\"` to be interpreted. + +```yaml +message: "Line one\nLine two" +``` + +- Use single quotes when you want literal text or when the string contains double quotes. +- Use double quotes when you need escape sequences or when the string contains single quotes and requires escaping. + +#### Multiline Strings + +Multiline strings are used for longer descriptions or scripts. + +Use `|` to preserve line breaks exactly as written. + +```yaml +description: | + This is a multi-line + description. + Line breaks are preserved. +``` + +Use `>` to fold lines into a single paragraph. + +```yaml +notes: > + This text will be folded + into a single line, + with spaces instead of line breaks. +``` + +- `|` preserves line breaks. +- `>` folds single line breaks into spaces. +- Blank lines are preserved when using `>`. +- Indentation under `|` or `>` must be consistent and becomes part of the string. + +#### Numbers + +YAML supports numeric scalar values without requiring quotes. +A number is interpreted as numeric when it contains only digits, with an optional sign, decimal point, or exponent. + +##### Valid Examples + +```yaml +positive_integer: 42 +negative_integer: -7 +decimal_number: 3.1415 +negative_decimal: -0.25 +scientific_notation: 1.2e6 +``` + +##### When to Use Quotes + +Numbers must be quoted if you want them treated as **strings**: + +```yaml +string_number: "42" +leading_zero_string: "007" +``` + +#### Null + +YAML supports a null value using any of the following forms: + +```yaml +value1: null +value2: Null +value3: NULL +value4: ~ +value5: +``` + +All of these are interpreted as `null`. + +### Lists (Sequences) + +Lists are introduced with a dash (`-`) at the start of each item. + +```yaml +tags: + - Light + - LivingRoom +``` + +Flow style is also supported: + +```yaml +tags: [Light, LivingRoom] +``` + +### Mappings (Key/Value Pairs) + +Mappings define key/value structures. + +```yaml +stateDescription: + config: + pattern: "%.1f °C" +``` + +Flow style is also supported: + +```yaml +stateDescription: { config: { pattern: "%.1f °C" } } +``` + +#### How YAML Identifies a Mapping Entry + +A mapping entry uses the pattern: + +```yaml +key: value +``` + +The space after the colon is required. + +```yaml +text: "Alice: Hello" # Must be quoted because the colon is followed by a space +url: http://example.com # Safe unquoted because the colon is not followed by a space +``` + +## Anchors and Aliases + +Anchors define reusable blocks. +Aliases reference those blocks. + +```yaml +defaults: &defaultConfig + refresh: 60 + geolocation: 1.23456, 7.890123 + +thing: + label: "Lamp" + config: *defaultConfig +``` + +For more details, see [Anchors and Aliases](anchors.md). + +## Merge Keys + +YAML merge keys use the `<<:` syntax to merge mappings. + +> **Note:** Although merge keys (`<<:`) are not defined in YAML 1.2, the YAML Composer includes full support for them. +> They are a foundational mechanism used throughout the preprocessing system to combine mappings, apply defaults, and build modular configuration structures. + +```yaml +base: &base + type: Switch + category: Light + +item: + <<: *base + label: "Desk Lamp" +``` + +Effective structure: + +```yaml +item: + type: Switch + category: Light + label: "Desk Lamp" +``` + +For more details, see [Merge Keys](merge-keys.md). + +## YAML Tags (Brief Introduction) + +YAML supports tags, which change how a value is interpreted. +Some tags are built in, and YAML Composer adds several **custom tags** such as `!include`, `!if`, and `!remove` to enable enhanced features. + +A tag can be applied to a simple scalar: + +```yaml +value: !include file.inc.yaml +``` + +A tag can also be applied to a mapping. +In that form, it applies to everything inside the mapping: + +```yaml +thing: !if + if: ${thing_enabled} + then: + label: Room Light +``` + +You will see these custom tags throughout the enhanced YAML configuration. + +## Common Pitfalls + +- Using tab characters instead of spaces for indentation. +- Using inconsistent indentation. +- Misaligning list items. +- Omitting quotes around strings containing special characters. +- Incorrectly nesting mappings. +- Incorrect indentation under `|` or `>` for multiline strings. + +If a YAML file fails to load, indentation, quoting, or multiline formatting issues are often the cause. diff --git a/bundles/org.openhab.io.yamlcomposer/doc/conditionals.md b/bundles/org.openhab.io.yamlcomposer/doc/conditionals.md new file mode 100644 index 0000000000..6369a1729a --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/doc/conditionals.md @@ -0,0 +1,164 @@ +# Conditionals (!if) + +The `!if` tag performs logical branching during the **preprocessing phase**. + +> **Note:** Conditions are evaluated **once** when the YAML file is loaded. +> These are not runtime rules. +> They do not react to live state changes in openHAB. + +[[toc]] + +## When to Use `!if` + +Use the `!if` tag to adapt your configuration based on the **Resolution Context** (variables defined in the file, injected via `!include` or `!insert`, or [environment globals](variables.md#env-to-access-environment-variables)). + +- **Conditional Snippets**: choose between alternative configuration blocks or values. +- **Optional Properties**: conditionally merge in additional settings using [merge keys (<<)](merge-keys.md). + +## Basic Syntax + +The `!if` tag supports two forms: a **Mapping Form** for simple logic and a **Sequence Form** for multiple branches. + +The `if:` and `elseif:` keys are treated as **implicit expressions**. +You do not need to wrap the expression in `${...}`. + +### Mapping Form (Simple) + +Use this for simple if/else decisions. + +```yaml +example: !if + if: env == 'prod' + then: "secure-server-url" + else: "localhost" +``` + +| Key | Description | Required | +|:-------|:------------------------------------------------------------------------------------------------------------------------|:---------| +| `if` | The expression to evaluate. | Yes | +| `then` | The value to return if **truthy** ([see rules below](#truthiness-rules)). Can be a scalar, map, list, or any valid tag. | Yes | +| `else` | The value to return if **falsy**. | No | + +### Sequence Form (Multiple Branches) + +Use this for multiple ordered conditions. + +Evaluates conditions in order and stops at the first **truthy** match. +If no condition matches and no `else` is provided, the tag resolves to `null`. + +```yaml +environment_type: !if + - if: hardware_version >= 2 + then: "high-power-mode" + - elseif: battery_powered + then: "eco-mode" + - else: "standard-mode" +``` + +## Expression Evaluation + +The `if:` key follows a specific order of operations. + +### 1. Bare Expressions (Recommended) + +The string is evaluated directly as an expression against the available variables. + +```yaml +if: count > 10 and status == 'ALARM' +``` + +::: tip +The expression can be quoted when it contains characters that YAML would otherwise misinterpret, such as `:` or `#`. +::: + +### 2. Using substitution pattern (Advanced — Double Evaluation) + +If you use a substitution pattern inside an `if:` key, the substitution engine runs **first** to resolve `${...}` patterns. +The resulting string is then evaluated as an expression. +This is useful for building logic strings from variables. + +```yaml +variables: + operator: ">" + +test: !if + if: 75 ${operator} 50 + # Step 1: ${operator} resolves to ">" + # Step 2: The if expression resolves to "75 > 50" + # Step 3: expression evaluates to true + then: "High" +``` + +## Truthiness Rules + +When a value is used in a conditional, it is first evaluated and then interpreted as either **truthy** or **falsy**. + +The following values are considered **falsy**: + +- `false` +- `null` +- `0` or `0.0` +- empty strings (`""`) +- empty lists (`[]`) +- empty maps (`{}`) + +All other values are **truthy**. +This includes any non‑empty string, any non‑zero number, and any non‑empty collection. + +### Short-Circuiting (Lazy Evaluation) + +Only the active branch is processed. +Tags such as `!include` inside inactive branches are ignored. +An `!include` in an inactive branch is never loaded and will not cause errors if the file does not exist. + +## Advanced Integration + +### Nesting and Composition + +The `!if` tag is fully recursive. +You can nest `!if` tags within the `then` or `else` blocks to create complex decision trees. + +```yaml +status: !if + if: device_online + then: !if + if: battery_level < 20 + then: "online-low-battery" + else: "online-healthy" + else: "offline" +``` + +### Conditional Merging (Mixins) + +Use `!if` with the YAML merge key (`<<`) to conditionally mix in sets of properties. + +```yaml +server_config: + port: 8080 + <<: !if + if: is_prod + then: + ssl_enabled: true + strict_security: true +``` + +### Using !include and !insert + +You can return entire files or templates by using `!include` or `!insert` inside a branch. +Only the tag in the active branch is processed. + +```yaml +network_settings: !if + if: wifi_enabled + then: !include wifi-config.inc.yaml + else: !insert ethernet-template +``` + +## Common Pitfalls + +1. **Expression vs String Literal**: `if: production` checks for a variable named `production`. + To check for the literal string, quote it: `if: env == 'production'`. +1. **Omitting `else`**: If no condition matches and there is no `else`, the result is `null`. +1. **Invalid YAML**: Even inactive branches must be syntactically valid YAML. + +See [Expression Syntax](variables.md#expression-syntax) for more details. diff --git a/bundles/org.openhab.io.yamlcomposer/doc/include.md b/bundles/org.openhab.io.yamlcomposer/doc/include.md new file mode 100644 index 0000000000..1cc249c5f8 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/doc/include.md @@ -0,0 +1,291 @@ +# Including Other Files + +`!include` inserts the referenced file or structure exactly at the position where the include appears. + +YAML Composer supports including external YAML files to facilitate modular, reusable, and maintainable configurations. +This is especially useful for modular reuse, creating device [packages](packages.md), or separating concerns across multiple files. + +[[toc]] + +## Syntax Options + +### Short Form + +```yaml +keyname: !include filename.inc.yaml +``` + +```yaml +keyname: !include filename.inc.yaml?arg1=value1&flag +``` + +The short form allows you to reference another file directly after the `!include` tag. +Optional arguments may be appended using URL‑style query syntax: + +- `arg1=value1` assigns a specific value +- `flag` (a value‑less argument) is interpreted as `true` +- Multiple arguments are separated with `&` +- The file name, argument names, and argument values are URL‑decoded. + If any of them contain characters such as `+`, `%`, or other reserved symbols, make sure to URL‑encode them. + +This form is ideal when you only need to include a file and pass a small number of simple parameters. + +### Long Form (supports variables) + +```yaml +# Block style (multi-line) +keyname: !include + file: filename.inc.yaml + vars: + var1: value1 + var2: value2 +``` + +```yaml +# Flow style (single-line) +keyname: !include { file: filename.inc.yaml, vars: { var1: value1, var2: value2 } } +``` + +The `vars:` mapping is layered on top of the file’s current variables to form the evaluation context. +In the long form, the `vars:` section is optional. + +::: tip Passing Existing Variables to Included Files + +The `vars:` section of an `!include` directive can contain literal values or references to **existing variables**. + +Example: + +```yaml +keyname: !include + file: xx.inc.yaml + vars: + var1: ${mainvar} +``` + +The include file can now refer to `var1` without relying on the variable names used in the main file. +This is especially useful when the same include file is shared across multiple configurations that may use different variable names. + +::: + +The contents of the include file are inserted as the value for the given key. +This means that the top-level keys in the include file become sub-keys of the key in the main file. + +## Variable Resolution Order + +Understanding how variables interact across files is important when using includes, especially when include files define their own defaults. + +When include files are involved, variables can originate from multiple sources. +Their values are resolved according to the following order, from highest priority to lowest priority. + +1. Inline `vars` in `!include` directives +1. Global `variables` defined in the main file +1. Local `variables` defined inside the include file + +Variables provided in the `vars` section of an `!include` directive are visible only within the included file, but they override both global and local variables. + +Local variables defined inside an include file can act as default values when they are not provided in the `!include` directive. +This allows include files to be reused with different parameters while keeping sensible defaults inside the file itself. + +**Example:** + +`main.yaml`: + +```yaml +variables: + broker: mqtt:broker:main + +things: + mqtt:topic:livingroom-window: !include + file: mqtt_contact.inc.yaml + vars: + label: Living Room Window + id: livingroom-window + + mqtt:topic:bedroom-window: !include + file: mqtt_contact.inc.yaml + vars: + label: Bedroom Window + id: bedroom-window + broker: mqtt:broker:external # override the global broker variable +``` + +`mqtt_contact.inc.yaml`: + +```yaml +bridge: ${broker} +label: ${label} +config: + availabilityTopic: ${id}/availability + payloadAvailable: online + payloadNotAvailable: offline +``` + +Resulting configuration: + +```yaml +things: + mqtt:topic:livingroom-window: + bridge: mqtt:broker:main + label: Living Room Window + config: + availabilityTopic: livingroom-window/availability + payloadAvailable: online + payloadNotAvailable: offline + + mqtt:topic:bedroom-window: + bridge: mqtt:broker:external + label: Bedroom Window + config: + availabilityTopic: bedroom-window/availability + payloadAvailable: online + payloadNotAvailable: offline +``` + +## File Naming & Reload Behavior + +### Include File Naming + +Include files should use a dedicated extension, either `.inc.yaml` or `.inc.yml`. +Files with a normal `.yaml` extension are treated as **main configuration files** and must contain the full [top‑level structure]({{base}}/configuration/yaml/) required by the YAML configuration schema. + +In contrast, `.inc.yaml` files are recognized as include fragments and are only processed when referenced through `!include`. + +::: tip Note +Include fragments and main files are intentionally treated as different roles. +Using the same file as both a compiled main file and an included fragment is not supported. +::: + +### Path Resolution + +Include file paths may be written as absolute paths or as paths relative to the current file. +YAML Composer also supports two shorthand prefixes that simplify referencing files inside the configuration directory. + +#### Shorthand Prefixes + +| Shorthand | Resolves To | Meaning | +|---------------------|-------------------------------------|----------------------------------------------------------------------------------------------------------------------------| +| `@/path`
`@path` | `${OPENHAB_CONF}/path` | The openHAB configuration root. Use when referencing files anywhere under `OPENHAB_CONF`. The slash after `@` is optional. | +| `$/path`
`$path` | `${OPENHAB_CONF}/yamlcomposer/path` | The source directory for your extended-syntax YAML files. The slash after `$` is optional. | + +##### `@/path` → `${OPENHAB_CONF}/path` + +A leading `@` resolves directly to the openHAB configuration root (`OPENHAB_CONF`). + +```yaml +key: !include "@/yaml/includes/device.inc.yaml" +# Resolves to: ${OPENHAB_CONF}/yaml/includes/device.inc.yaml +``` + +**Notes:** + +- Paths beginning with `@` **must be quoted** (e.g., `"@/path"`), because YAML plain scalars cannot begin with `@`. +- The slash after `@` is **optional**: `"@yaml/includes/device.inc.yaml"` works the same. +- Use `@` when you want to reference files anywhere under the configuration root without writing long absolute paths. + +##### `$/path` → `${OPENHAB_CONF}/yamlcomposer/path` + +A leading `$` resolves to `OPENHAB_CONF/yamlcomposer`. + +**Example directory layout:** + +```sh +OPENHAB_CONF/ + yamlcomposer/ + shared.inc.yaml + lights/ + kitchen/ + main.yaml +``` + +If `main.yaml` contains: + +```yaml +key: !include "$/shared.inc.yaml" +``` + +Then `$` resolves to: + +```sh +${OPENHAB_CONF}/yamlcomposer +``` + +So the final resolved path becomes: + +```sh +${OPENHAB_CONF}/yamlcomposer/shared.inc.yaml +``` + +**Notes:** + +- The slash after `$` is **optional**: `"$shared.inc.yaml"` works the same. +- It avoids brittle paths like `../../../shared.inc.yaml`. + +#### Relative Paths + +If the path does not begin with `/`, `@`, or `$` it is interpreted as a path **relative to the directory of the including file**. +You may use `.` and `..` to refer to the current and parent directory. + +**Example directory layout:** + +```sh +yamlcomposer/ + parent.inc.yaml + main/ + main.yaml + shared.inc.yaml + common/ + defaults.inc.yaml +``` + +**Same directory:** + +```yaml +key: !include "shared.inc.yaml" +# Resolves to: yamlcomposer/main/shared.inc.yaml +``` + +**Navigate downward (into a subdirectory):** + +```yaml +key: !include "common/defaults.inc.yaml" +# Resolves to: yamlcomposer/main/common/defaults.inc.yaml +``` + +**Navigate upward:** + +```yaml +key: !include "../parent.inc.yaml" +# Resolves to: yamlcomposer/parent.inc.yaml +``` + +Relative paths always resolve from the directory containing the including file. + +#### Using Variable Substitutions + +You can use variable substitution patterns in the file name to construct paths dynamically. +This includes normal variables, [predefined variables](variables.md#predefined-variables), and environment variables exposed through `ENV`. + +**Example:** + +```yaml +keyname: !include + file: ${OPENHAB_CONF}/packages/hue-light.pkg.yaml +``` + +### File Organization + +It may be helpful to store include files in a dedicated subdirectory. +These files can be referenced using relative paths, the `@` or `$` shorthands, or full absolute paths. +Choose whichever style best matches your preference. + +### Nested Includes + +Include files may themselves contain `!include` directives. +This allows configurations to be composed from multiple layers: + +main.yaml → a.inc.yaml → b.inc.yaml → … + +### Reload Behavior + +When an include file changes, YAML Composer reloads the main files that reference it rather than attempting to load the include file directly. +Reloads occur only if the include file is located within `${OPENHAB_CONF}/yamlcomposer/`. diff --git a/bundles/org.openhab.io.yamlcomposer/doc/merge-keys.md b/bundles/org.openhab.io.yamlcomposer/doc/merge-keys.md new file mode 100644 index 0000000000..41dc6959b6 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/doc/merge-keys.md @@ -0,0 +1,205 @@ +# Merge Keys + +Merge keys (`<<`) let you combine mappings defined directly in a mapping with other mappings defined elsewhere, such as: + +- [Anchors](anchors.md) +- [!include](include.md) (External files) +- [!insert](templates.md) (In-file Templates) +- [!if](conditionals.md) (Conditionals) +- [Variables](variables.md) + +They promote reusability and avoid repetition by letting you define common mappings once and update them in a single place. + +[[toc]] + +## What Merge Keys Do + +A merge key takes one or more source mappings and merges their key–value pairs into the current mapping. +The source mappings may be written inline, referenced through an anchor, or loaded from another file. +The merged values behave exactly as if they were written inline. +Local fields may add to or override the merged content. + +## Merge Rules + +- Only mappings can be merged. +- Merge keys may appear multiple times or as a list. +- Merge order matters because keys from earlier mappings override keys from later ones. +- Local keys always override merged keys, even when defined after a merge key. +- Merges are shallow, and nested mappings are replaced rather than combined. + +### Example: Multiple Merges and Precedence + +```yaml +.defaults: &DEFAULTS + icon: default + autoupdate: false + +.metadata: &LIGHTS + tags: [Light] + icon: light + +items: + Lamp: + <<: [*LIGHTS, *DEFAULTS] + label: Desk Lamp +``` + +Result: + +```yaml +items: + Lamp: + label: Desk Lamp + icon: light # from LIGHTS (earlier mapping wins) + autoupdate: false + tags: [Light] +``` + +## Limitations + +- Merge keys only merge mappings, not lists. +- Merge keys cannot modify scalar values. +- Merge keys cannot be used inside sequences unless the sequence elements are mappings. + +## Integrating with Other Features + +### Anchors + +[Anchors](anchors.md) define reusable structures whose content can be inserted into the current mapping via an alias. +Merge keys then combine that anchored content with any local fields in the mapping. +Anchors are a standard YAML feature for sharing data within a single file. +Variables or templates are often preferred for more complex needs. + +```yaml +items: + Light1: &LIGHT_BASE + type: Switch + icon: light + autoupdate: false + label: Light One + + Light2: + <<: *LIGHT_BASE + label: Light Two +``` + +See also: [Using Hidden Keys for Anchors](anchors.md#using-hidden-keys-for-anchors). + +### `!include` + +The [!include](include.md) tag loads the contents of another file so it can be inserted into the current mapping. +Merge keys then combine the included content with any local fields in the mapping. + +**Benefits over anchors:** + +- Included files can be parameterized, allowing the same structure to be reused with different values. +- Included files can be shared across many different YAML files, whereas anchors are limited to a single file. + +```yaml +# light-common.inc.yaml +power: + type: switch + stateTopic: ${id}/power +availability: + type: contact + stateTopic: ${id}/availability +``` + +```yaml +# light-color.inc.yaml +color: + type: color + stateTopic: ${id}/color +``` + +```yaml +# main.yaml +things: + mqtt:topic:living-room-light: + channels: + <<: !include { file: light-common.inc.yaml, vars: { id: living-room-light } } + <<: !include { file: light-color.inc.yaml, vars: { id: living-room-light } } +``` + +### Templates (`!insert`) + +The [!insert](templates.md) tag inserts the contents of an in-file template into the current mapping. +Merge keys then combine the template’s fields with any local fields in that mapping. + +Templates can be parameterized when used in a merge key. + +```yaml +templates: + common_channels: + power: + type: switch + stateTopic: ${id}/power + availability: + type: contact + stateTopic: ${id}/availability + + color_channel: + color: + type: color + stateTopic: ${id}/color + +things: + mqtt:topic:living-room-light: + channels: + <<: !insert { template: common_channels, vars: { id: living-room-light } } + <<: !insert { template: color_channel, vars: { id: living-room-light } } +``` + +### Conditionals (`!if`) + +The [!if](conditionals.md) tag allows you to choose between different mappings or return `null` to skip the merge entirely. +The engine resolves tags before the merge, so the merge key receives the final mapping. + +```yaml +# Merge extra settings only for production +server: + port: 8080 + <<: !if + if: is_prod + then: { ssl: true, cache: true } +``` + +### Substitution + +Merge keys can combine content produced dynamically from an expression. +This enables inline conditional merging using Python-style expressions. + +```yaml +variables: + has_color: true + color_channel: + color: + type: color + +things: + mqtt:topic:light: + channels: + power: + type: switch + # Returns color_channel map if true + <<: ${color_channel if has_color} +``` + +::: tip Hints + +- Merge keys only merge mappings, so the expression must resolve to a map. +- Avoid compound patterns such as `${foo}${bar}` or `x${foo}` because they are interpreted as literal strings and cannot be merged. +- Merging `null` or an empty map is a no-op, effectively omitting the merge key. + +::: + +## Related Topics + +For details on how variables are resolved and how substitution interacts with YAML structures, see [Variables & Substitution — Return Types & Coercion](variables.md#return-types--coercion). + +## Best Practices + +- Use merge keys to define reusable base templates. +- Keep anchors inside hidden keys to avoid clutter. +- Use includes or packages to centralize shared structures. +- Prefer simple, predictable structures for reusable blocks. diff --git a/bundles/org.openhab.io.yamlcomposer/doc/packages.md b/bundles/org.openhab.io.yamlcomposer/doc/packages.md new file mode 100644 index 0000000000..307ae00989 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/doc/packages.md @@ -0,0 +1,450 @@ +# Packages + +Packages provide a way to bundle multiple related YAML sections into a reusable, parameterized unit. +Unlike fragment-level insertion, a package expands into full top‑level sections such as `things:` or `items:`. +These sections may come from an external file or a same‑file template. +They are then merged into the current configuration. + +[[toc]] + +## Purposes + +- **Logical Grouping:** + Packages allow a **Thing** and its related **Items**, channels, and metadata to be defined together in one file, representing a complete, self‑contained device definition. + +- **Reuse with Different Parameters:** + Through variable substitution, a single package can be instantiated multiple times with different values. + This makes it easy to define many similar devices such as sensors, lights, or switches from one shared template. + +## Package Syntax and Structure + +Packages are declared in the main YAML file under the top‑level `packages:` section. +Each entry defines a package ID and the source from which the package content is obtained. + +```yaml +packages: + : + : + ... +``` + +### Key Components + +- **Package ID:** + A package ID is a unique identifier for the package. + It may include spaces. + The special variable `${package_id}` resolves to this key inside the package content. + +- **Package Source:** + A package can be created from either of the following sources: + + - **`!include` (external file):** + Loads a separate YAML file and applies the package’s variable context to it. + See the [!include syntax options](include.md#syntax-options). + + - **`!insert` (same‑file template):** + Expands a template defined under the main file’s `templates:` section. + See the [!insert syntax options](templates.md#syntax-options). + + Both forms support parameterization through `vars:` and participate fully in package merging. + +## Package Source Contents + +- **Top‑Level Sections:** + Package sources contain any combination of top‑level keys such as `things:` and `items:`. + +- **Uniqueness:** + Because package sources can be referenced multiple times, use variable substitutions such as `${package_id}` and unique `vars:` variables for entity UIDs in each invocation to avoid collisions. + +- **Deep Nesting:** + A package source may itself include other packages. + When a package references another package via `packages:` inside an included file, it creates an inheritance chain. + Properties merge downward sequentially from the deepest core file out to the final main file. + +## Package Example + +`main.yaml`: + +```yaml +variables: + broker: mqtt:broker:main + +packages: + livingroom-light: !include + file: package/mqtt-light.inc.yaml + vars: + name: Living_Room_Light + label: Living Room Light + + bedroom-light: !include + file: package/mqtt-light.inc.yaml + vars: + name: Bed_Room_Light + label: Bed Room Light +``` + +`package/mqtt-light.inc.yaml`: + +```yaml +things: + mqtt:topic:${package_id}: + bridge: ${broker} + channels: + power: + type: switch + config: + stateTopic: ${package_id}/state + commandTopic: ${package_id}/set/state + # ... other channels (brightness, color) + +items: + ${name}_Power: + type: Switch + label: ${label} Power + channel: mqtt:topic:${package_id}:power + # ... more items for the light, e.g. brightness, color, etc. +``` + +Resulting YAML structure: + +```yaml +things: + mqtt:topic:livingroom-light: + bridge: mqtt:broker:main + channels: + power: + type: switch + config: + stateTopic: livingroom-light/state + commandTopic: livingroom-light/set/state + mqtt:topic:bedroom-light: + bridge: mqtt:broker:main + channels: + power: + type: switch + config: + stateTopic: bedroom-light/state + commandTopic: bedroom-light/set/state + +items: + Living_Room_Light_Power: + type: Switch + label: Living Room Light Power + channel: mqtt:topic:livingroom-light:power + Bed_Room_Light_Power: + type: Switch + label: Bed Room Light Power + channel: mqtt:topic:bedroom-light:power +``` + +## Merge Behavior + +### Final Top-Level Sections + +A final top‑level section is the fully expanded `things:`, `items:`, or other top‑level section of the configuration that openHAB receives after all packages, templates, includes, and merges have been applied. +Entries defined directly under these sections can merge with package‑generated entries when their identifiers match. +Entries remain independent when their identifiers do not match. + +::: tip Note +The following example uses `!insert`, but the same merge rules apply to packages sourced from `!include`. +::: + +When a package is expanded, its contents are merged into the main YAML structure. +You may customize the resulting structure by overriding, adding, or removing elements defined in the package. +This is done by redefining the elements you want to customize in the main file. +These redefinitions appear in the final top‑level section. + +### Default Merge Behavior + +Source YAML File: + +```yaml +templates: + number_item: + items: + ${package_id}_Item: + type: Number + label: Package Label + tags: [Measurement] + metadata: + stateDescription: + config: + min: 1 + pattern: '%.3f' + widget: + value: oh-card + +packages: + Number: !insert number_item + +# This is the final top‑level `items:` section of the configuration +# The packages will merge into this section +items: + Number_Item: + label: Power Draw + dimension: Power + tags: [Power] + metadata: + stateDescription: + config: + max: 10 +``` + +Result: + +```yaml +items: + Number_Item: + type: Number + label: Power Draw + dimension: Power + tags: + - Measurement + - Power + metadata: + stateDescription: + config: + min: 1 + max: 10 + pattern: '%.3f' + widget: + value: oh-card +``` + +The way keys interact depends on their data type. + +| Data Type | Behavior | Description | +|-----------|-----------|----------------------------------------------------------------------------------------------------------| +| Scalar | Overwrite | A scalar in the final top‑level section replaces the scalar defined at the same path inside the package. | +| Map | Merge | Maps are merged key by key, recursively. | +| List | Merge | Lists are concatenated (package values first) and de-duplicated. | + +### Automatic Removal of Empty Values + +During merging, empty structures are automatically stripped from the final configuration. +Empty maps (`{}`) and lists (`[]`) as well as map keys whose value is `null` or an empty string are removed. +This keeps the resulting configuration clean and allows packages to define catch‑all defaults. + +**Example:** + +```yaml +variables: + icon: null # default to avoid unknown‑variable warnings + +icon: ${icon} +``` + +Because `icon` evaluates to `null`, the entire `icon:` key is removed from the merged output unless the including file overrides it. + +### How Package Merging Differs from YAML Merge Keys + +Mappings from packages are merged recursively with the corresponding mappings in the final top‑level section. +This differs from standard YAML merge keys, which perform only shallow merges. + +**Merge Key (shallow merge):** + +```yaml +# merge key: +targetkey: + foo: + bar: + boo: baz + <<: # merge `foo` into `targetkey` + foo: + bar: + boo: waldo + goo: fy + qux: quux +``` + +```yaml +# result — the merge key's foo mapping +# is ignored because foo already exists in main +targetkey: + foo: + bar: + boo: baz +``` + +**Package Merging (recursive merge):** + +```yaml +# main file +targetkey: + foo: + bar: + boo: baz + +packages: + anyid: !include packagefile.inc.yaml +``` + +```yaml +# packagefile.inc.yaml +targetkey: + foo: + bar: + boo: waldo + goo: fy + qux: quux +``` + +```yaml +# result: +targetkey: + foo: + bar: + boo: baz # main file overrides matching keys + goo: fy # but includes additional keys... + qux: quux # from the package +``` + +recursive merging allows customization at any depth in the mapping. + +### Controlling Package Merge Behavior with Tags + +Use these special YAML tags to explicitly override the default recursive merge behavior. +These directives can be declared at any point in the configuration pipeline—either within the **main YAML file** or deeply embedded inside an intermediate **include file** acting as a middle-tier package. + +#### 1. The `!replace` Tag + +The `!replace` tag forces an absolute replacement for maps or lists that would otherwise merge. +This acts as a destructive splice, completely discarding any existing inherited data structure beneath that key from upstream sources and starting fresh with the new layout provided. + +#### 2. The `!remove` Tag + +The `!remove` tag cleanly deletes the corresponding key and its entire sub-tree from the configuration hierarchy. +It is ideal for pruning specific components, metadata keys, or configurations exposed by a baseline package that do not apply to the current downstream context. + +--- + +### Example: Nested Package Overrides + +When packages include other packages, a middle-tier file can surgically modify or strip elements declared by the core baseline configuration before it ever reaches the main composition file. + +#### Core Configuration (`nested-items.inc.yaml`) + +Provides a core item layout with deep metadata fields: + +```yaml +items: + target_item: + label: base_label + tags: + - from_nested + metadata: + stateDescription: + value: from_nested + category: + value: from_nested + untouched_item: + label: untouched_from_nested +``` + +#### Middle-Tier Override Package (`pkg-with-overrides.inc.yaml`) + +This file consumes the core configuration via `!include`, but applies `!replace` and `!remove` tags to selectively adjust properties: + +```yaml +packages: + nested: !include nested-items.inc.yaml +items: + target_item: + tags: !replace # 1. Overwrite the list, discarding 'from_nested' + - from_outer + metadata: + stateDescription: !remove # 2. Entirely purge this sub-tree key + category: !replace # 3. Swap out the map structure with a fresh one + value: from_outer + config: + origin: nested_package_override +``` + +#### Root Configuration (`main.yaml`) + +Loads the middle-tier package: + +```yaml +packages: + p1: !include pkg-with-overrides.inc.yaml +``` + +#### Final Merged Output + +When evaluated by the preprocessor, the downstream overrides successfully neutralize the deep properties inherited from the original package file. +The `stateDescription` block is erased, lists and maps are replaced cleanly, and unrelated siblings are passed through untouched: + +```yaml +items: + target_item: + label: base_label + tags: + - from_outer + metadata: + category: + value: from_outer + config: + origin: nested_package_override + untouched_item: + label: untouched_from_nested +``` + +::: tip Usage Notes + +- `!replace` and `!remove` are evaluated locally within each package layer before that package's content is returned and merged into the caller. + This allows intermediate include files to cleanly intercept, prune, or completely reset configurations inherited from deeper core packages. +- They can be declared in the root configuration file or anywhere down an inheritance chain of nested package includes. +- `!remove` target references apply to entire map keys; individual array elements within a list cannot be targeted for independent removal. +- Use `!replace` on an array to drop inherited elements and start an array list with entirely new contents. + +::: + +## Strategic Use of Package IDs + +Choose a **Package ID** that can also serve as a Thing UID fragment, Item name, or similar identifier. +This avoids defining extra variables in your package source and lets you derive related identifiers directly from `${package_id}`. + +You can override `${package_id}` in the `vars:` block if needed. + +**Example:** + +```yaml +# main file +packages: + Living_Room_Light: !include light.inc.yaml + Kitchen_Light: !include light.inc.yaml +``` + +```yaml +# light.inc.yaml package source +variables: + id: ${package_id|lower|replace('_', '-')} + thing_uid: "mqtt:topic:${id}" + item_name: ${package_id} + label: ${package_id|replace('_', ' ')} +``` + +**Resulting variables:** + +| Variable | Living_Room_Light | Kitchen_Light | +|-----------------|--------------------------------|----------------------------| +| `${package_id}` | `Living_Room_Light` | `Kitchen_Light` | +| `${id}` | `living-room-light` | `kitchen-light` | +| `${thing_uid}` | `mqtt:topic:living-room-light` | `mqtt:topic:kitchen-light` | +| `${item_name}` | `Living_Room_Light` | `Kitchen_Light` | +| `${label}` | `Living Room Light` | `Kitchen Light` | + +## Limitation: Top-Level Merge Keys in `packages:` + +Top-level YAML merge keys (for example `<<:`) inside the `packages:` map are not supported. +Each package must be declared explicitly as a direct key under `packages:`. + +The following pattern is **not supported**: + +```yaml +packages: + <<: !include common-packages.yaml +``` + +In this form, the merge key tries to inject package declarations into `packages:` itself. +The composer does not expand package declarations through top-level merge keys. diff --git a/bundles/org.openhab.io.yamlcomposer/doc/templates.md b/bundles/org.openhab.io.yamlcomposer/doc/templates.md new file mode 100644 index 0000000000..458effa052 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/doc/templates.md @@ -0,0 +1,188 @@ +# Templates + +Templates let you define reusable blocks of YAML directly within the same file. +They are stored under the top‑level `templates:` section and can be expanded using the `!insert` tag. + +Templates behave similarly to `!include` in how they are evaluated — including variable scoping and per‑invocation isolation — but they are defined locally rather than in external files. + +[[toc]] + +## What Templates Are + +A template is a named YAML node stored under the top‑level `templates:` section. +A template may contain **any YAML value**: + +- a mapping +- a sequence +- a scalar + +The keys under `templates:` (`simple_greeting`, `greeting`, `channel_block`, `color_list`) are **template names**. +They identify each template and are not part of the template’s content. + +Example: + +```yaml +templates: + simple_greeting: Hello! + greeting: Hello, ${name}! + channel_block: + power: + type: switch + stateTopic: ${id}/power + color_list: + - red + - green + - blue +``` + +::: tip Important! + +Templates are not evaluated when defined. +They are evaluated **each time** they are expanded with `!insert`. +Therefore, a template may contain references to variables that do not yet exist; those variables are supplied later during the `!insert` invocation. + +::: + +## Syntax Options + +`!insert` expands a template at the point where it appears. +It supports two forms: + +### Short Form + +```yaml +message: !insert simple_greeting +``` + +```yaml +message: !insert simple_greeting?arg1=value1&flag +``` + +The short form allows you to reference a template directly after the `!insert` tag. +Optional arguments may be appended using URL‑style query syntax: + +- `arg1=value1` assigns a specific value +- `flag` (a value‑less argument) is interpreted as `true` +- Multiple arguments are separated with `&` +- The template name, argument names, and argument values are URL‑decoded. + If any of them contain characters such as `+`, `%`, or other reserved symbols, make sure to URL‑encode them. + +Use the no‑argument form when you simply want to insert a template as‑is. +Use the query‑parameter form when you want to pass a small number of simple parameters. + +### Long Form (supports variables) + +```yaml +# Block style (multi-line) +message: !insert + template: greeting + vars: + name: "Jimmy" + +# Or flow style (single-line) +message: !insert { template: greeting, vars: { name: "Jimmy" } } +``` + +The `vars:` mapping is layered on top of the file’s current variables to form the evaluation context. +In the long form, the `vars:` section is optional. + +## How Template Evaluation Works + +Each `!insert` call is evaluated in its **own isolated context**: + +- It begins with all variables defined in the current file. +- If this file was loaded via `!include`, it also inherits variables from parent files. +- Variables provided through `vars:` override or extend this context. +- The template body is evaluated using this combined set of variables. +- The resulting value — scalar, list, or mapping — replaces the `!insert` node. + +### Example: Scalar Template + +```yaml +templates: + greeting: Hello, ${name}! + +message: !insert { template: greeting, vars: { name: "Alice" } } +``` + +Result: + +```yaml +message: Hello, Alice! +``` + +### Example: Template Producing a Mapping + +```yaml +templates: + channel_block: + power: + type: switch + stateTopic: ${id}/power + +thing: + channels: !insert { template: channel_block, vars: { id: living-room-light } } +``` + +### Example: Template as a List Item + +```yaml +templates: + color_entry: ${color} + +colors: + - !insert { template: color_entry, vars: { color: red } } + - !insert { template: color_entry, vars: { color: green } } + - !insert { template: color_entry, vars: { color: blue } } +``` + +### Example: Template Producing a Sequence + +```yaml +templates: + rgb: + - red + - green + - blue + +palette: !insert rgb +``` + +## Similarities with `!include` + +Templates and includes share the same evaluation model: + +- Each invocation is evaluated in its own isolated context. +- The context includes: + - variables defined in the current file + - variables inherited from parent files (if this file was loaded via `!include`) + - variables provided through `vars:` for this specific invocation +- Variable resolution inside the template uses this combined context. + +This section is intentionally mechanical — it explains _how_ templates behave, not _when_ to use them. + +## Template Scope and Variable Resolution + +Inside a template: + +- Interpolation happens only when the template is inserted. +- `${var}` resolves using the combined variable context described above. +- Each `!insert` call is isolated — variables from one invocation do not leak into another. + +## Limitations + +- Templates must be defined under the top‑level `templates:` section. +- Templates cannot be shared across files (use `!include` for that). +- Template names must be unique within the file. +- A template may produce any YAML value, but the resulting value must be valid for the location where it is inserted (scalar, list item, or mapping field). + +## Best Practices + +- Keep template names descriptive and consistent. +- Prefer the block style when passing multiple variables for readability. +- Use the composed output in `OPENHAB_CONF/yaml/composed/` to verify template expansion. + +--- + +In‑file templates provide a flexible, parameterizable way to structure reusable YAML within a single file. +They complement `!include` and anchors, offering a clean mechanism for building consistent configurations. diff --git a/bundles/org.openhab.io.yamlcomposer/doc/variables.md b/bundles/org.openhab.io.yamlcomposer/doc/variables.md new file mode 100644 index 0000000000..19a76e6723 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/doc/variables.md @@ -0,0 +1,281 @@ +# Variables & Substitution + +Variables allow you to define reusable values and substitute them throughout your YAML configuration. +They provide the flexibility needed for complex templates and reduce hard-coded values. +These variables work consistently across both the current file and any included packages. + +[[toc]] + +## Variable Definition + +Variables are defined within a top-level `variables:` section. +It is recommended to place this section at the beginning of the file for better organization. + +**Example:** + +```yaml +variables: + # Scalar variables + expire: 5m + label: Living Room + + # Map variable + mqtt: + broker: mqtt:broker:mybroker + + # List variable + rooms: + - Kitchen + - Bedroom +``` + +## Variable Substitution + +### Default Substitution Behavior + +Substitution enables dynamic value construction instead of static hard-coding. +Whenever a scalar contains a `${...}` pattern, the preprocessor evaluates the expression and replaces the pattern with the result. +This makes it easy to construct labels, identifiers, and paths from defined variables. +Substitution also applies to YAML keys to allow identifiers to be built dynamically. + +**Example:** + +```yaml +variables: + room: Kitchen + light_id: Kitchen_Light + +items: + ${light_id}: + label: ${room} Light +``` + +**Resulting Output:** + +```yaml +items: + Kitchen_Light: + label: Kitchen Light +``` + +### Return Types & Coercion + +The output type depends on how the substitution is structured within the YAML scalar. + +#### Type Preservation (Single Expression) + +If a YAML value consists **entirely** of a single substitution pattern, the resulting object preserves the original Java type returned by the expression. +This allows you to inject complex structures like booleans, lists, or maps directly. + +```yaml +is_active: ${status == 'ON'} # Becomes a real Boolean +target_rooms: ${rooms} # Becomes a real List +connection: "${mqtt_config_map}" # Becomes a real Map +``` + +#### String Coercion (Mixed Content) + +If the substitution pattern is combined with any other text, or if multiple patterns are used together, the entire value is coerced into a **String**. + +```yaml +description: "Status is ${status}" # String: "Status is ON" +concatenated: "${10}${20}" # String: "1020" +room_name: "${room} " # String: "Kitchen " (includes space) +``` + +| Syntax Pattern | Resulting Type | Example Output | +|:---------------|:---------------|:------------------------| +| `${expr}` | **Preserved** | `[Item1, Item2]` (List) | +| `"${expr}"` | **Preserved** | `true` (Boolean) | +| `Text ${expr}` | **String** | `"Count: 5"` (String) | +| `${ex1}${ex2}` | **String** | `"1020"` (String) | + +### The `!literal` Tag and `!sub` Escape Hatch + +The `!literal` tag disables substitution recursively for a specific YAML node. +This is useful when you need to preserve the `${...}` syntax as literal text. +Inside a `!literal` section, you can use the `!sub` tag to re-enable substitution for a specific child node. +When these tags overlap, the innermost tag always controls the final behavior. + +**Example:** + +```yaml +top: !literal + foo: ${LITERAL} + bar: + baz: ${LITERAL} + quux: !sub ${substituted} + grault: ${LITERAL} +``` + +## Expression Syntax + +The Expression syntax is based on the [Jinja expression](https://jinja.palletsprojects.com/en/stable/templates/#expressions) language. +Only expressions inside `${...}` are supported; template blocks like `{% if %}` or `{% for %}` are not available. + +### Variable References + +1. `label`: Refers to a scalar variable. +1. `rooms[0]`: Refers to the first element of a list. +1. `mqtt.broker` or `mqtt['broker']`: Refers to a map subkey. +1. `mqtt[key]`: Resolves the key dynamically using the value of the `key` variable. + +### Operations & Concatenation + +An expression can include string, arithmetic, and boolean operations. + +1. **String Concatenation**: Use the `~` operator (e.g., `"Room " ~ index`). +1. **Coercion**: The `~` operator is the preferred way to join values because it automatically converts non-string operands into strings. +1. **Automatic Joining**: Adjacent literal text and substitution patterns are automatically joined without operators (e.g., `value: "Hello ${username}"`). + +> **Note:** Referencing an undefined variable resolves to `null`, logs a warning, and results in an empty string if used in a string context. + +### List Concatenation with `+` + +Jinja’s `+` operator supports list concatenation. +If one side is a list and the other is a scalar, the scalar is automatically wrapped into a single-element list. + +**Example:** + +```yaml +variables: + groups: [Group1, Group2] + location: SemanticLocationGroup + +effective_groups: ${ groups + location } +# Result: [Group1, Group2, SemanticLocationGroup] +``` + +### Built-in Filters + +Filters are applied using the `variable|filter` syntax and can be chained. + +#### Text Transformation + +| Filter | Description | +|:-------------|:------------------------------------------------------------| +| `capitalize` | Capitalize a value. | +| `title` | Return a titlecased version. | +| `lower` | Convert a value to lowercase. | +| `upper` | Convert a value to uppercase. | +| `replace` | Replace a substring. | +| `trim` | Strip leading and trailing characters (default whitespace). | + +#### Formatting + +| Filter | Description | +|:---------|:----------------------------------------------| +| `format` | Apply values to a printf-style format string. | +| `round` | Round a number to an optional precision. | +| `int` | Convert a value into an integer. | + +#### Collection Helpers + +| Filter | Description | +|:---------|:---------------------------------------| +| `first` | Return the first item of a list. | +| `length` | Return the length of a list or string. | + +#### Fallbacks + +| Filter | Description | +|:----------|:--------------------------------------------------------------| +| `default` | Return a default value if the variable is empty or undefined. | + +**Default Example:** + +```yaml +label: ${room_label | default('Kitchen')} +``` + +### Custom Filters + +| Filter | Description | +|:--------|:------------------------------------------------------------------------------------| +| `label` | Converts identifiers (camelCase, snake_case) into human-friendly titles. | +| `dig` | Safely navigates deep maps; returns `null` instead of an error if a key is missing. | + +**`dig` Example:** + +```yaml +# Dot notation and mixed access supported +user: ${ infrastructure | dig('config.login.user') } +host: ${ VARS | dig('config', 'servers', 1, 'host') | default('localhost') } +``` + +## Advanced Usage + +### Predefined Variables + +The Composer injects environmental and file-system context automatically. +These variables can be interpolated just like regular ones and are helpful when constructing paths for directives. + +| Variable | Description | +|:-------------------|:------------------------------------------------------------------------| +| `OPENHAB_CONF` | Absolute path to openHAB's main configuration directory. | +| `OPENHAB_USERDATA` | Absolute path to openHAB's userdata directory. | +| `__FILE__` | Absolute path to the current file. | +| `__FILE_NAME__` | Filename portion without the extension or leading path. | +| `__FILE_EXT__` | File extension portion of the current file name. | +| `__DIRECTORY__` | Directory portion of the current file. | +| `__DIR__` | Alias for `__DIRECTORY__`. | +| `package_id` | Automatically resolved to the Package ID within included package files. | + +### Handling Reserved Keywords + +If a variable name is a Jinja keyword (like `and`, `or`, `if`), access it via the `VARS` dictionary. +This also works for variable names containing characters like hyphens that are invalid in direct references. + +```yaml +foo: ${VARS['and']} +``` + +### ENV to Access Environment Variables + +A special variable `ENV` exposes a map of system environment variables. +This is especially useful for configurations running within Docker containers. + +```yaml +mode: ${ENV.OPENHAB_MODE} # Resolves to the environment value +``` + +### Calling Java Methods + +Variables retain their Java types, allowing you to call standard methods directly. + +Common types you may encounter include: + +- [String](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/String.html) +- [Integer](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/Integer.html) +- [Double](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/Double.html) +- [Boolean](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/Boolean.html) +- [Map](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/Map.html) +- [List](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/util/List.html) + +This is useful for logic beyond built-in filters, such as complex regex replacements. + +```yaml +# Use Java String methods for complex logic +is_sensor: ${ device_id.startsWith("sensor_") } +clean_id: ${ device_id.replaceAll("ABC", "XYZ") } +``` + +### Custom Pattern Delimiters + +If your content conflicts with `${...}`, you can define custom delimiters using a named pattern. +This allows the same pattern to be used consistently across all included files. + +```yaml +variables: + jinja: "{{..}}" + +foo: !sub:jinja "Hello {{ username }}!" +``` + +## Common Pitfalls + +1. **Unquoted Operators**: Expressions with operators should be quoted so YAML doesn't misinterpret characters. +1. **Reserved Names**: Avoid naming variables using keywords like `true`, `false`, `null`, `in`, or `if`. +1. **`+` vs `~`**: Use `~` for strings to avoid type mismatch errors and use `+` for numbers or lists. +1. **Jinja Blocks**: Template blocks such as `{% for %}` are not supported; use inline `if` expressions instead. +1. **Whitespace Sensitivity**: Spaces outside of quotes inside `${ ... }` are ignored, but spaces in quoted strings are preserved. diff --git a/bundles/org.openhab.io.yamlcomposer/pom.xml b/bundles/org.openhab.io.yamlcomposer/pom.xml new file mode 100644 index 0000000000..e47b84c528 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/pom.xml @@ -0,0 +1,100 @@ + + + + 4.0.0 + + + org.openhab.addons.bundles + org.openhab.addons.reactor.bundles + 5.2.0-SNAPSHOT + + + org.openhab.io.yamlcomposer + + openHAB Add-ons :: Bundles :: IO :: YAML Composer + + + com.sun.jdi;resolution:=optional,com.sun.jdi.connect;resolution:=optional,com.sun.jdi.event;resolution:=optional,com.sun.jdi.request;resolution:=optional,com.sun.tools.attach;resolution:=optional,jdk.internal.module;resolution:=optional,sun.misc;resolution:=optional,* + + + + + org.snakeyaml + snakeyaml-engine + 3.0.1 + + + org.openhab.osgiify + com.hubspot.jinjava.jinjava + 2.7.4 + compile + + + com.google.re2j + re2j + 1.2 + compile + + + com.google.guava + guava + 33.5.0-jre + test + + + org.openhab.osgiify + com.hubspot.immutables.immutables-exceptions + 1.9 + compile + + + org.apache.commons + commons-lang3 + 3.20.0 + compile + + + org.jsoup + jsoup + 1.15.3 + compile + + + commons-net + commons-net + ${commons.net.version} + compile + + + ch.obermuhlner + big-math + 2.3.2 + compile + + + com.googlecode.java-ipv6 + java-ipv6 + 0.17 + compile + + + com.google.code.findbugs + jsr305 + 3.0.2 + compile + + + org.javassist + javassist + 3.30.2-GA + compile + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + ${jackson.version} + compile + + + diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/feature/feature.xml b/bundles/org.openhab.io.yamlcomposer/src/main/feature/feature.xml new file mode 100644 index 0000000000..cd68cd9cfe --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/feature/feature.xml @@ -0,0 +1,15 @@ + + + mvn:org.openhab.core.features.karaf/org.openhab.core.features.karaf.openhab-core/${ohc.version}/xml/features + + + openhab-runtime-base + openhab.tp-commons-net + mvn:org.snakeyaml/snakeyaml-engine/3.0.1 + mvn:org.openhab.osgiify/com.hubspot.jinjava.jinjava/2.7.4 + mvn:org.openhab.osgiify/com.google.re2j.re2j/1.2 + mvn:ch.obermuhlner/big-math/2.3.2 + mvn:org.openhab.osgiify/com.hubspot.immutables.immutables-exceptions/1.9 + mvn:org.openhab.addons.bundles/org.openhab.io.yamlcomposer/${project.version} + + diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/BufferedLogger.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/BufferedLogger.java new file mode 100644 index 0000000000..5743df3193 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/BufferedLogger.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import java.util.Objects; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.slf4j.Logger; +import org.slf4j.helpers.MessageFormatter; + +/** + * A wrapper/decorator for SLF4J loggers that intercepts warnings for + * consolidation through {@link LogSession} while passing other + * log levels through immediately. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class BufferedLogger { + private final Logger delegate; + private final LogSession session; + + public BufferedLogger(Logger delegate, LogSession session) { + this.delegate = delegate; + this.session = session; + } + + public LogSession getLogSession() { + return session; + } + + /** + * Intercepts warnings and sends them to the session for count consolidation. + */ + public void warn(String template, Object... args) { + String resolvedMessage = Objects.requireNonNull(MessageFormatter.arrayFormat(template, args).getMessage()); + session.trackWarning(delegate, resolvedMessage); + } + + // Pass-through methods for immediate logging + public void info(String format, Object... args) { + delegate.info(format, args); + } + + public void debug(String format, Object... args) { + delegate.debug(format, args); + } + + public void trace(String format, Object... args) { + delegate.trace(format, args); + } + + public void error(String format, Object... args) { + delegate.error(format, args); + } + + // Simple string overloads + public void info(String msg) { + delegate.info(msg); + } + + public void debug(String msg) { + delegate.debug(msg); + } + + public void trace(String msg) { + delegate.trace(msg); + } + + public void error(String msg) { + delegate.error(msg); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/ComposerConfig.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/ComposerConfig.java new file mode 100644 index 0000000000..86ea24ba02 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/ComposerConfig.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import java.nio.file.Path; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.openhab.core.OpenHAB; + +/** + * Static configuration constants and system path resolution for the YAML Composer. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public final class ComposerConfig { + + private static final Path DEFAULT_CONFIG_ROOT = normalizeAbsolute(Path.of(OpenHAB.getConfigFolder())); + private static final Path DEFAULT_USERDATA_ROOT = normalizeAbsolute(Path.of(OpenHAB.getUserDataFolder())); + + private static volatile Path configRootOverride = DEFAULT_CONFIG_ROOT; + private static volatile Path userDataRootOverride = DEFAULT_USERDATA_ROOT; + + public static final Path SOURCE_ROOT_DIRECTORY = Path.of("yamlcomposer"); + public static final Path OUTPUT_ROOT_DIRECTORY = Path.of("yaml", "composed"); + + // Special section keys + public static final String TEMPLATES_KEY = "templates"; + public static final String VARIABLES_KEY = "variables"; + public static final String PACKAGES_KEY = "packages"; + + // Preprocessing limits + public static final int MAX_INCLUDE_DEPTH = 100; + + private ComposerConfig() { + // Static utility class + } + + public static Path configRoot() { + return configRootOverride; + } + + public static Path userDataRoot() { + return userDataRootOverride; + } + + public static Path sourceRoot() { + return configRoot().resolve(SOURCE_ROOT_DIRECTORY); + } + + public static Path outputRoot() { + return configRoot().resolve(OUTPUT_ROOT_DIRECTORY); + } + + static void setRootsForTesting(Path configRoot, Path userDataRoot) { + configRootOverride = normalizeAbsolute(configRoot); + userDataRootOverride = normalizeAbsolute(userDataRoot); + } + + static void resetRootsForTesting() { + configRootOverride = DEFAULT_CONFIG_ROOT; + userDataRootOverride = DEFAULT_USERDATA_ROOT; + } + + private static Path normalizeAbsolute(Path path) { + return path.toAbsolutePath().normalize(); + } + + /** + * Resolves the output path for a generated YAML file. + * + * @param sourcePath source file's absolute path + * @return the path where the generated output should be written + */ + public static Path resolveOutputPath(Path sourcePath) { + Path relativeToSourceRoot = sourceRoot().relativize(sourcePath); + return outputRoot().resolve(relativeToSourceRoot); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/ComposerUtils.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/ComposerUtils.java new file mode 100644 index 0000000000..50a0757802 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/ComposerUtils.java @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.core.OpenHAB; +import org.openhab.io.yamlcomposer.internal.constructors.ModelConstructor; +import org.snakeyaml.engine.v2.api.Dump; +import org.snakeyaml.engine.v2.api.DumpSettings; +import org.snakeyaml.engine.v2.api.Load; +import org.snakeyaml.engine.v2.api.LoadSettings; +import org.snakeyaml.engine.v2.common.FlowStyle; +import org.snakeyaml.engine.v2.resolver.ScalarResolver; +import org.snakeyaml.engine.v2.schema.CoreSchema; +import org.snakeyaml.engine.v2.schema.Schema; + +/** + * Pure utility functions for YAML preprocessing. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +final class ComposerUtils { + private static final String GENERATED_HEADER = """ + # ============================================================================== + # Generated by openHAB %s (%s) YAML Composer, DO NOT EDIT + # Source: %s + # Generated: %s + # ============================================================================== + """; + + private ComposerUtils() { + // Static utility class + } + + /** + * Removes top-level keys that start with a dot from the given map. + * + * @param dataMap the map to process + */ + static void removeHiddenKeys(Map dataMap) { + dataMap.keySet().removeIf(key -> key instanceof String keyStr && keyStr.startsWith(".")); + } + + /** + * Writes the fully processed YAML representation as it would be seen by openHAB after preprocessing + * to a compiled output file. + * + * @param dataMap the resulting data map to dump + * @param sourcePath the absolute path of the source file being processed + * @throws IOException if writing to the file fails + */ + static void writeCompiledOutput(Object dataMap, Path sourcePath, Path outputPath) throws IOException { + + Path outputDir = outputPath.getParent(); + if (outputDir != null) { + Files.createDirectories(outputDir); + } + + DumpSettings dumpSettings = DumpSettings.builder() // + .setDefaultFlowStyle(FlowStyle.BLOCK) // + .setIndentWithIndicator(true) // + .setIndicatorIndent(2) // + .setMultiLineFlow(true) // + .build(); + Dump yaml = new Dump(dumpSettings); + Object unaliasedData = breakAliases(dataMap); + String compiledYaml = yaml.dumpToString(unaliasedData); + compiledYaml = formatYamlSpacing(compiledYaml); + + Path sourcePathRelative = ComposerConfig.configRoot().relativize(sourcePath); + compiledYaml = GENERATED_HEADER.formatted(OpenHAB.getVersion(), OpenHAB.buildString(), sourcePathRelative, + ZonedDateTime.now()) + "\n" + compiledYaml; + + Files.writeString(outputPath, compiledYaml, StandardCharsets.UTF_8); + } + + /** + * Creates a new SnakeYAML Engine load instance with the custom composer constructor and resolver. + * + * @param sourcePath the source path of the YAML file for error reporting in the constructor + * @return a new Load instance + */ + private static Load createYamlLoader(String sourcePath) { + Schema schema = new CoreSchema() { + @Override + public ScalarResolver getScalarResolver() { + return new ModelResolver(); + } + }; + + LoadSettings loadSettings = LoadSettings.builder() // + .setLabel(sourcePath) // + .setAllowDuplicateKeys(false) // + .setUseMarks(true) // + .setSchema(schema) // + .build(); + return new Load(loadSettings, new ModelConstructor(loadSettings, sourcePath)); + } + + /** + * Loads YAML content from the given byte array using a Yaml instance configured with the custom constructor. + * + * @param fileBytes the YAML content as a byte array + * @param sourcePath the source path of the YAML file for error reporting in the constructor + * @return the loaded YAML object + * @throws IOException if an I/O error occurs + */ + static @Nullable Object loadYaml(byte[] fileBytes, Path sourcePath) throws IOException { + Load loader = createYamlLoader(sourcePath.toString()); + return loader.loadFromInputStream(new ByteArrayInputStream(fileBytes)); + } + + /** + * Produces a deep-copy of the common YAML structure objects (maps, lists, sets) + * such that no container instance is shared more than once in the resulting + * graph. This removes object identity sharing that SnakeYAML would otherwise + * emit as anchors/aliases when dumping. + * + * @param root the loaded YAML object (may be Map, List, Set, scalar, or custom) + * @return a deep-copied object graph with sharing broken + */ + private static @Nullable Object breakAliases(@Nullable Object root) { + if (root == null) { + return null; + } + return cloneNode(root, new IdentityHashMap<>()); + } + + /** + * Post-processes a YAML string to inject empty lines for improved readability. + * + *

+ * This method applies two specific spacing rules: + *

    + *
  • Level 0: Inserts a newline before every top-level key (except the first line).
    • + *
  • Level 2: Inserts a newline between sibling nodes indented by exactly two spaces.
    • + *
+ * + *

+ * The logic ensures that the first child of a section remains "glued" to its parent header, + * and it safely handles both Map keys (e.g., {@code Key:}) and List items (e.g., {@code - Item}). + * + * @param yaml The raw YAML string to be formatted. + * @return A formatted YAML string with injected "breathing room." + */ + private static @Nullable String formatYamlSpacing(@Nullable String yaml) { + if (yaml == null || yaml.isEmpty()) { + return yaml; + } + + String[] lines = yaml.split("\n"); + StringBuilder sb = new StringBuilder(); + + boolean hasSeenLevel0 = false; + boolean hasSeenLevel2 = false; + + for (String line : lines) { + // --- Level 0 Logic (Sections) --- + // Matches lines starting with a character (not space, not comment) + if (!line.startsWith(" ") && !line.startsWith("#")) { + if (hasSeenLevel0) { + sb.append("\n"); + } + hasSeenLevel0 = true; + hasSeenLevel2 = false; // Reset level 2 tracker for the new section + } + + // --- Level 2 Logic (Items or List Elements) --- + // Matches exactly 2 spaces followed by a non-whitespace character. + // This targets siblings within a top-level section. + else if (line.matches("^ {2}[^ \\t].*")) { + if (hasSeenLevel2) { + sb.append("\n"); + } + hasSeenLevel2 = true; + } + + sb.append(line).append("\n"); + } + + return sb.toString().trim(); + } + + /** + * Recursively clones a YAML node, breaking object identity to prevent SnakeYAML from emitting anchors/aliases. + * + * @param node The YAML node to clone. + * @param stack The recursion stack used to handle cycles. + * @return A deep-cloned YAML node with object identity broken. + */ + private static @Nullable Object cloneNode(@Nullable Object node, IdentityHashMap stack) { + if (node == null) { + return null; + } + // Scalars and non-container objects are returned as-is + if (!(node instanceof Map) && !(node instanceof List) && !(node instanceof Set)) { + return node; + } + + // If the node is already on the current recursion stack, return the + // partially-built copy to handle cycles. We do NOT reuse completed + // copies for separate occurrences — that would preserve sharing and + // produce aliases. + if (stack.containsKey(node)) { + return stack.get(node); + } + + if (node instanceof Map) { + @SuppressWarnings("unchecked") + Map<@Nullable Object, @Nullable Object> original = (Map<@Nullable Object, @Nullable Object>) node; + Map<@Nullable Object, @Nullable Object> copy = new LinkedHashMap<>(original.size()); + stack.put(node, copy); + for (Map.Entry<@Nullable Object, @Nullable Object> e : original.entrySet()) { + Object k = e.getKey(); + Object v = e.getValue(); + Object newKey = cloneNode(k, stack); + Object newVal = cloneNode(v, stack); + copy.put(newKey, newVal); + } + // Remove from stack so later separate occurrences are cloned again + stack.remove(node); + return copy; + } + + if (node instanceof List) { + @SuppressWarnings("unchecked") + List<@Nullable Object> original = (List<@Nullable Object>) node; + List<@Nullable Object> copy = new ArrayList<>(original.size()); + stack.put(node, copy); + for (@Nullable + Object item : original) { + copy.add(cloneNode(item, stack)); + } + stack.remove(node); + return copy; + } + + // Set (preserve insertion order if possible) + if (node instanceof Set) { + @SuppressWarnings("unchecked") + Set<@Nullable Object> original = (Set<@Nullable Object>) node; + Set<@Nullable Object> copy = new LinkedHashSet<>(original.size()); + stack.put(node, copy); + for (Object item : original) { + copy.add(cloneNode(item, stack)); + } + stack.remove(node); + return copy; + } + + // Fallback: return node as-is + return node; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/IncludeRegistry.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/IncludeRegistry.java new file mode 100644 index 0000000000..34d9f5be68 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/IncludeRegistry.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * The {@link IncludeRegistry} manages a bidirectional association between + * main files and the include files they reference. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public final class IncludeRegistry { + private final Map> mainToIncludes = new HashMap<>(); + private final Map> includeToMains = new HashMap<>(); + + @SuppressWarnings("null") + public synchronized void registerInclude(Path mainFile, Path include) { + mainToIncludes.computeIfAbsent(mainFile, key -> new HashSet<>()).add(include); + includeToMains.computeIfAbsent(include, key -> new HashSet<>()).add(mainFile); + } + + public synchronized void removeMain(Path mainFile) { + Set includes = mainToIncludes.remove(mainFile); + if (includes == null) { + return; + } + includes.forEach(include -> { + Set mains = includeToMains.get(include); + if (mains != null) { + mains.remove(mainFile); + if (mains.isEmpty()) { + includeToMains.remove(include); + } + } + }); + } + + public synchronized Set getMainsForInclude(Path include) { + return new HashSet<>(includeToMains.getOrDefault(include, Set.of())); + } + + public synchronized Set getIncludesForMain(Path mainFile) { + return new HashSet<>(mainToIncludes.getOrDefault(mainFile, Set.of())); + } + + public synchronized boolean hasInclude(Path include) { + Set mains = includeToMains.get(include); + return mains != null && !mains.isEmpty(); + } + + public synchronized void clear() { + mainToIncludes.clear(); + includeToMains.clear(); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/LogSession.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/LogSession.java new file mode 100644 index 0000000000..f03876275c --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/LogSession.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.slf4j.Logger; + +/** + * Coordinates log aggregation across multiple classes and instances during a single + * processing session. + *

+ * It collects warnings and their counts, then flushes them with a summary suffix + * (e.g., " (3 times)") when the session is closed. This prevents log flooding + * during recursive operations like YAML !includes. + *

+ * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class LogSession implements AutoCloseable { + private final Map counts = new LinkedHashMap<>(); + private final Map loggers = new HashMap<>(); + + /** + * Records a warning message for later consolidation. + */ + public void trackWarning(Logger logger, String resolvedMessage) { + loggers.putIfAbsent(resolvedMessage, logger); + counts.merge(resolvedMessage, 1, Integer::sum); + } + + /** + * Returns an unmodifiable list of tracked warning messages in the order they were recorded. + * Primarily used for verification in unit tests. + */ + List getTrackedWarnings() { + return List.copyOf(counts.keySet()); + } + + public int getTotalWarningCount() { + return counts.values().stream().mapToInt(Integer::intValue).sum(); + } + + /** + * Issues all buffered warnings to their respective loggers with occurrence counts. + */ + public void flush() { + counts.forEach((msg, count) -> { + String output = msg; + if (count > 1) { + String separator = msg.contains("\n") ? System.lineSeparator() : " "; + output = msg + separator + "(" + count + " times)"; + } + Objects.requireNonNull(loggers.get(msg)).warn(output); + }); + counts.clear(); + loggers.clear(); + } + + @Override + public void close() { + flush(); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/ModelResolver.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/ModelResolver.java new file mode 100644 index 0000000000..02550aea15 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/ModelResolver.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import java.util.regex.Pattern; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.openhab.io.yamlcomposer.internal.constructors.ModelConstructor; +import org.snakeyaml.engine.v2.nodes.Tag; +import org.snakeyaml.engine.v2.resolver.CoreScalarResolver; + +/** + * A custom scalar resolver for SnakeYAML Engine that shifts structural processing + * from the engine's native layer to our internal composer layer. + * + *

+ * This implementation modifies the default SnakeYAML Engine resolution in two key ways: + *

+ *
    + *
  • Custom Merge Key Handling: It disables the default merge key ({@code <<}) + * resolution. Instead, it registers a custom implicit resolver that maps the + * merge key to a unique tag. This allows the composer to handle object + * merging manually, providing more control over the transformation lifecycle.
    • + * + *
  • Collision Prevention: It opts out of the default {@code ENV_TAG} + * resolver. This ensures that the standard YAML environment variable syntax + * ({@code ${VAR}}) does not interfere with our custom substitution syntax.
    • + *
+ * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +class ModelResolver extends CoreScalarResolver { + ModelResolver() { + super(false); + addImplicitResolver(ModelConstructor.DEFERRED_MERGE_TAG, MERGE, "<"); + } + + @Override + @NonNullByDefault({}) + public void addImplicitResolver(Tag tag, Pattern regexp, String first) { + if (!Tag.ENV_TAG.equals(tag)) { + super.addImplicitResolver(tag, regexp, first); + } + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/StringInterpolator.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/StringInterpolator.java new file mode 100644 index 0000000000..8dc86b8234 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/StringInterpolator.java @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.expression.ExpressionEvaluator; + +/** + * The {@link StringInterpolator} provides utility methods for performing + * variable substitution and expression evaluation in YAML processing. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class StringInterpolator { + static final String DEFAULT_BEGIN = "${"; + static final String DEFAULT_END = "}"; + + /** + * Matches ${...} interpolation patterns in YAML strings. + * + * Allows nested quotes like default('{foo}') without prematurely stopping at + * the inner '}' + * + * Examples: + * - ${name} : Simple variable substitution + * - ${ name|capitalize } : Apply capitalize filter + * - ${ name|default('value') } : Use default if variable is not set + * - ${ name|upper } : Convert to uppercase + */ + public static final Pattern DEFAULT_SUBSTITUTION_PATTERN = Pattern.compile(""" + \\$\\{ + (? + (?: + "[^"]*" | + '[^']*' | + \\{[^}]*?\\} | + [^}] + )*? + ) + \\} + """, Pattern.COMMENTS | Pattern.DOTALL); + + private StringInterpolator() { + // Utility class, no instances + } + + /** + * Compiles a substitution pattern from begin and end delimiters. + * + * @param begin the opening delimiter (e.g., "${") + * @param end the closing delimiter (e.g., "}") + * @return the compiled pattern + */ + public static Pattern compileSubstitutionPattern(String begin, String end) { + if (DEFAULT_BEGIN.equals(begin) && DEFAULT_END.equals(end)) { + return DEFAULT_SUBSTITUTION_PATTERN; + } + String quotedBegin = Pattern.quote(begin); + String quotedEnd = Pattern.quote(end); + // Allow quoted segments to contain the end delimiter; otherwise stop at the + // first closing delimiter. + String content = """ + "[^"]*"|'[^']*'|\\{[^}]*?\\}|(?:(?!%s).) + """.formatted(quotedEnd).strip(); + String regex = quotedBegin + "(?(" + content + ")*?)" + quotedEnd; + return Pattern.compile(regex, Pattern.DOTALL); + } + + /** + * Compiles a substitution pattern from a pattern specification string with format + * {@code ..}. + * + * @param patternSpec the pattern specification, e.g. {@code $[[..]]} + * @return the compiled pattern, or {@code null} when the specification is invalid + */ + public static @Nullable Pattern compilePatternSpec(String patternSpec) { + int separator = patternSpec.indexOf(".."); + if (separator <= 0 || separator >= patternSpec.length() - 2) { + return null; + } + + String begin = patternSpec.substring(0, separator); + String end = patternSpec.substring(separator + 2); + if (begin.isEmpty() || end.isEmpty()) { + return null; + } + + return compileSubstitutionPattern(begin, end); + } + + /** + * Evaluates a variable value, performing substitution if needed. + * Returns the native object type if the entire value is a single placeholder, + * otherwise returns a string with interpolated values. + * + * @param value the raw string value that may contain placeholder patterns to substitute + * @param pattern the substitution pattern to use + * @param variables the variable map + * @param logSession the logging session + * @param sourceLocation description of the source location for logging + * @return the evaluated value + */ + public static @Nullable Object interpolate(String value, Pattern pattern, Map variables, + LogSession logSession, String sourceLocation) { + Matcher matcher = pattern.matcher(value); + if (!matcher.find()) { + return value; + } + + // If the whole value is exactly one ${...}, evaluate it and return the native object. + // This preserves non-string types (maps, lists, numbers, booleans) + // when the value is a plain placeholder. + if (matcher.matches()) { + String content = Objects.requireNonNull(matcher.group("content")); + return evaluateExpression(content, variables, logSession, sourceLocation); + } + + // Evaluate the expressions inside the ${...} patterns and replace them in the string. + String interpolated = matcher.replaceAll(match -> { + String content = Objects.requireNonNull(match.group("content")); + Object result = evaluateExpression(content, variables, logSession, sourceLocation); + String rendered = result != null ? result.toString() : ""; + return Matcher.quoteReplacement(rendered); + }); + + return interpolated; + } + + /** + * Evaluates a Jinjava expression. + * + * When an error occurs during evaluation, and null is returned. + * A warning is logged and consolidated through logSession + * + * @param expression the expression to evaluate without the ${} delimiters + * @param variables the variable context for evaluation + * @param logSession the logging session for warnings and errors during evaluation + * @param sourceLocation description of the source location for logging + * @return the evaluated result + */ + public static @Nullable Object evaluateExpression(String expression, Map variables, + LogSession logSession, String sourceLocation) { + Object rendered = ExpressionEvaluator.renderObject(expression, variables, logSession, sourceLocation); + return rendered; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/YamlComposer.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/YamlComposer.java new file mode 100644 index 0000000000..f060bb3cba --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/YamlComposer.java @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.core.PackageProcessor; +import org.openhab.io.yamlcomposer.internal.core.ProcessingPhase; +import org.openhab.io.yamlcomposer.internal.core.RecursiveTransformer; +import org.openhab.io.yamlcomposer.internal.core.SourceLocator; +import org.openhab.io.yamlcomposer.internal.core.TemplateLoader; +import org.openhab.io.yamlcomposer.internal.core.VariableLoader; +import org.openhab.io.yamlcomposer.internal.placeholders.SubstitutionPlaceholder; +import org.openhab.io.yamlcomposer.internal.processors.IfProcessor; +import org.openhab.io.yamlcomposer.internal.processors.IncludeProcessor; +import org.openhab.io.yamlcomposer.internal.processors.InsertProcessor; +import org.openhab.io.yamlcomposer.internal.processors.RemoveProcessor; +import org.openhab.io.yamlcomposer.internal.processors.ReplaceProcessor; +import org.openhab.io.yamlcomposer.internal.processors.SubstitutionProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.snakeyaml.engine.v2.exceptions.Mark; +import org.snakeyaml.engine.v2.exceptions.MarkedYamlEngineException; +import org.snakeyaml.engine.v2.exceptions.YamlEngineException; + +/** + * The {@link YamlComposer} is a utility class to load YAML files + * and process them into a final YAML structure that openHAB can use. + * + * The following features are supported: + * + *
    + *
  • YAML Anchors and aliases. + *
  • YAML Merge keys (<<) to allow merging of maps with override semantics. + *
  • Variable substitution and interpolation using ${var} syntax. + *
  • Conditional evaluation using !if tag with simple boolean logic. + *
  • !include tag for including other YAML files. + *
  • !insert tag for inserting template content with local variable substitution. + *
  • Combining elements using packages. + *
+ * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class YamlComposer { + private static final Logger RAW_LOGGER = LoggerFactory.getLogger(YamlComposer.class); + + private final BufferedLogger logger; + + public static record CacheEntry(byte[] bytes, long mtime) { + } + + private final Path absolutePath; + private final Path relativePath; + + private final Map variables; + private final Map templates; + + private final Set includeStack; + private final ConcurrentHashMap includeCache; + + private final RecursiveTransformer recursiveTransformer; + + /** + * Constructs a YamlComposer for the given file path and context. + * + * @param path the file path for resolving relative includes + * @param variables initial variable context + * @param includeStack current include stack for circular reference detection + * @param includeCallback callback invoked for each included file + * @param logSession the log session for warning consolidation + * @param includeCache the cache for included files + * @throws YamlEngineException if a circular include is detected or if the maximum include depth is exceeded + */ + public YamlComposer(Path path, Map variables, Set includeStack, + Consumer includeCallback, LogSession logSession, + ConcurrentHashMap includeCache) { + this.absolutePath = Objects.requireNonNull(path.toAbsolutePath().normalize()); + this.relativePath = ComposerConfig.configRoot().relativize(absolutePath); + this.logger = new BufferedLogger(RAW_LOGGER, logSession); + this.variables = new HashMap<>(variables); + this.includeCache = includeCache; + this.templates = new HashMap<>(); + + // Validate circular inclusion and depth before processing + Set newIncludeStack = new LinkedHashSet<>(includeStack); + if (!newIncludeStack.add(absolutePath)) { + @SuppressWarnings("null") + String includeStackChain = newIncludeStack.stream().map(Path::toString).collect(Collectors.joining(" -> ")); + throw new YamlEngineException( + "Circular inclusion detected: %s -> %s".formatted(includeStackChain, absolutePath)); + } + + if (newIncludeStack.size() > ComposerConfig.MAX_INCLUDE_DEPTH) { + throw new YamlEngineException("Maximum include depth (" + ComposerConfig.MAX_INCLUDE_DEPTH + ") exceeded"); + } + + this.includeStack = newIncludeStack; + + this.recursiveTransformer = new RecursiveTransformer(this.variables); + + this.recursiveTransformer.register(new SubstitutionProcessor(logger)); + this.recursiveTransformer.register(new IfProcessor(logger)); + this.recursiveTransformer.register( + new IncludeProcessor(absolutePath.getParent(), newIncludeStack, includeCallback, includeCache, logger)); + this.recursiveTransformer.register(new InsertProcessor(templates, logger)); + this.recursiveTransformer.register(new RemoveProcessor()); + this.recursiveTransformer.register(new ReplaceProcessor()); + } + + /** + * Loads a YAML file from the given {@link Path} and processes it through the + * full composer pipeline. + *

+ * This is the main entry point for the YAML Composer. It reads the file, + * parses the YAML, and applies all supported composer features. + *

+ * The {@code includeCallback} is invoked for each file referenced via an + * include directive, allowing the caller to track include usage so it can + * refresh models when included files change. + *

+ * The returned value is the fully evaluated Java object representation of the + * YAML document after all the processing steps have been applied. + * + * @param path the path to the YAML file to load and process; also used as + * the base directory for resolving relative includes + * @param includeCallback a callback invoked for each included file + * @return the processed Java object representation of the YAML file + * @throws IOException if the file cannot be read or if processing fails + */ + public static @Nullable Object load(Path path, Consumer includeCallback) throws IOException { + // Create a LogSession autocloseable object. It consolidates warnings and duplicates. + // Upon exit, any warnings will be logged. + try (LogSession session = new LogSession()) { + ConcurrentHashMap cache = new ConcurrentHashMap<>(); + return load(path, includeCallback, session, cache); + } + } + + /** + * Internal method to allow passing in a LogSession so we can manage it externally in tests. + * + * @param path the file path for resolving relative includes + * @param includeCallback callback invoked for each included file + * @param logSession the LogSession to use for logging warnings during loading + * @param includeCache the cache for included files to optimize repeated loads + * @return the processed Java object representation of the YAML file + * @throws IOException if there is an error reading or processing the YAML + */ + static @Nullable Object load(Path path, Consumer includeCallback, LogSession logSession, + ConcurrentHashMap includeCache) throws IOException { + Path absolutePath = path.toAbsolutePath().normalize(); + Path relativePath = ComposerConfig.configRoot().relativize(absolutePath); + try { + YamlComposer composer = new YamlComposer(absolutePath, Map.of(), Set.of(), includeCallback, logSession, + includeCache); + Object result = composer.load(); + + // Print a summary of warnings before the LogSession outputs all the warnings. + int totalWarnings = logSession.getTotalWarningCount(); + if (totalWarnings > 0) { + int unique = logSession.getTrackedWarnings().size(); + String issuesLabel = (unique == 1) ? "unique issue" : "unique issues"; + String warningLabel = (totalWarnings == 1) ? "warning" : "warnings"; + + RAW_LOGGER.warn("YAML Composer {}: Preprocessing completed with {} {} ({} {}).", relativePath, + totalWarnings, warningLabel, unique, issuesLabel); + } + + return result; + } catch (MarkedYamlEngineException e) { + String errorMsg = e.getMessage(); + Mark mark = e.getProblemMark().orElse(null); + if (mark != null) { + String location = "%d:%d".formatted(mark.getLine() + 1, mark.getColumn() + 1); + String errorClass = e.getClass().getSimpleName(); + errorMsg = "\n%s:%s %s %s".formatted(relativePath, location, errorClass, e.getMessage()); + } + throw new IOException(errorMsg, e); + } catch (YamlEngineException e) { + throw new IOException(e.getMessage(), e); + } + } + + /** + * Internal load method that performs the actual loading and processing of the YAML file. + * + * @return the processed Java object representation of the YAML file + * @throws IOException if there is an error reading the YAML file + * @throws YamlEngineException if there is an error during YAML parsing or processing + */ + public @Nullable Object load() throws IOException, YamlEngineException { + logger.debug("Loading file({}): {} with given vars {}", includeStack.size(), absolutePath, variables); + + // Phase 1: Parse YAML and initialize helper objects + byte[] yamlBytes = readYamlBytes(); + SourceLocator locator = new SourceLocator(yamlBytes); + + // Phase 2: set up initial variables + VariableLoader variableLoader = new VariableLoader(variables, absolutePath, recursiveTransformer, logger); + variableLoader.setSpecialVariables(); + + // Phase 3: load and parse YAML + Object yamlObj = ComposerUtils.loadYaml(yamlBytes, relativePath); + if (!(yamlObj instanceof Map)) { + yamlObj = recursiveTransformer.transform(yamlObj, ProcessingPhase.STANDARD); + + if (!(yamlObj instanceof Map)) { + return yamlObj; + } + } + + Map yamlMap = (Map) yamlObj; + + // Phase 4: extract variables and templates + Object variablesSection = removeByScalarKey(yamlMap, ComposerConfig.VARIABLES_KEY); + variableLoader.extractVariables(variablesSection, locator); + + // Extract templates because we want to defer substitutions in the templates + // until the template is instantiated with !insert, so that the variable context + // includes any variables passed in the !insert directive. + Object templatesSection = removeByScalarKey(yamlMap, ComposerConfig.TEMPLATES_KEY); + new TemplateLoader(logger, relativePath, templates, recursiveTransformer, locator) + .extractTemplates(templatesSection); + + // Phase 5: extract/remove packages from the main data because we want to + // inject the package_id into each package context + @Nullable + Object packagesObj = removeByScalarKey(yamlMap, ComposerConfig.PACKAGES_KEY); + + // Phase 6: Resolve merge keys and process substitutions, conditionals, includes and inserts + // in a single pass so that merge keys can merge data produced by includes/inserts. + yamlMap = recursiveTransformer.transform(yamlMap, ProcessingPhase.STANDARD); + + // Phase 7: process and merge packages + new PackageProcessor(logger, absolutePath, relativePath, locator, recursiveTransformer).mergePackages(yamlMap, + packagesObj); + + // Phase 8: process override placeholders (!replace, !remove) + yamlMap = recursiveTransformer.transform(yamlMap, ProcessingPhase.PACKAGE_OVERRIDES); + + // Phase 9: final cleanup and optional compiled output + ComposerUtils.removeHiddenKeys(yamlMap); + + return yamlMap; + } + + /** + * Reads the YAML file bytes with caching based on file modification time to optimize repeated loads of the same + * file. The bytes are cached in {@link #includeCache}. + * + * @return the YAML file bytes + * @throws IOException if there is an error reading the file + */ + private byte[] readYamlBytes() throws IOException { + CacheEntry cached = includeCache.get(absolutePath); + long currentMtime = Files.getLastModifiedTime(absolutePath).toMillis(); + + if (cached != null && cached.mtime == currentMtime) { + return cached.bytes; + } + + byte[] yamlBytes = Files.readAllBytes(absolutePath); + includeCache.put(absolutePath, new CacheEntry(yamlBytes, currentMtime)); + return yamlBytes; + } + + /** + * Checks if the given file name is an include file based on its extension. + * An include file ends with .inc.yml or .inc.yaml. + * + * @param fileName the name of the file to check + * @return true if it's an include file, false otherwise + */ + public static boolean isIncludeFile(String fileName) { + return fileName.endsWith(".inc.yml") || fileName.endsWith(".inc.yaml"); + } + + /** + * Checks if the given file name is a Yaml file based on its extension. + * A Yaml file ends with .yml or .yaml. + * + * @param fileName the name of the file to check + * @return true if it's a Yaml file, false otherwise + */ + public static boolean isYamlFile(String fileName) { + return fileName.endsWith(".yml") || fileName.endsWith(".yaml"); + } + + private static @Nullable Object removeByScalarKey(Map map, String key) { + @SuppressWarnings("unchecked") + Map<@Nullable Object, @Nullable Object> mutableMap = (Map<@Nullable Object, @Nullable Object>) map; + + for (Iterator> iterator = mutableMap.entrySet() + .iterator(); iterator.hasNext();) { + Map.Entry<@Nullable Object, @Nullable Object> entry = iterator.next(); + @Nullable + Object entryKey = entry.getKey(); + + if (entryKey == null) { + continue; + } + + boolean keyMatches = switch (entryKey) { + case String s -> key.equals(s); + case SubstitutionPlaceholder p -> key.equals(p.value()); + default -> false; + }; + + if (keyMatches) { + Object value = entry.getValue(); + iterator.remove(); + return value; + } + } + return null; + } + + public Path getAbsolutePath() { + return absolutePath; + } + + public Path getRelativePath() { + return relativePath; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/YamlComposerWatchService.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/YamlComposerWatchService.java new file mode 100644 index 0000000000..5f5bb4e112 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/YamlComposerWatchService.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Set; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.core.service.WatchService; +import org.openhab.core.service.WatchService.Kind; +import org.osgi.service.component.annotations.Activate; +import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Deactivate; +import org.osgi.service.component.annotations.Reference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Watches OPENHAB_CONF/yamlcomposer and writes composed YAML files to OPENHAB_CONF/yaml/composed. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +@Component(immediate = true, service = YamlComposerWatchService.class) +public class YamlComposerWatchService implements WatchService.WatchEventListener { + private static final Path WATCHED_SOURCE_PATH = ComposerConfig.SOURCE_ROOT_DIRECTORY; + + private static final String OUTPUT_ROOT_README = """ + This directory contains YAML files generated by the YAML Composer add-on. + Do not edit these files manually, as they will be overwritten by the composer. + Instead, edit the source YAML files in the 'yamlcomposer' directory and let the composer generate the output here. + """; + + private final Logger logger = LoggerFactory.getLogger(YamlComposerWatchService.class); + private final IncludeRegistry includeRegistry = new IncludeRegistry(); + private final WatchService watchService; + + @Activate + public YamlComposerWatchService(@Reference(target = WatchService.CONFIG_WATCHER_FILTER) WatchService watchService) { + this.watchService = watchService; + + try { + Path outputRoot = ComposerConfig.outputRoot(); + boolean brandNew = !Files.exists(outputRoot); + + Files.createDirectories(ComposerConfig.sourceRoot()); + Files.createDirectories(outputRoot); + + if (brandNew) { + Files.writeString(outputRoot.resolve("readme.txt"), OUTPUT_ROOT_README); + } + processDirectory(ComposerConfig.sourceRoot()); + } catch (IOException e) { + logger.warn("Cannot prepare yaml composer directories: {}", e.getMessage()); + } + + watchService.registerListener(this, WATCHED_SOURCE_PATH); + } + + @Deactivate + public void deactivate() { + watchService.unregisterListener(this); + includeRegistry.clear(); + } + + @Override + public synchronized void processWatchEvent(Kind kind, Path fullPath) { + Path sourcePath = fullPath; + if (kind != Kind.DELETE && (Files.isDirectory(sourcePath) || !Files.isReadable(sourcePath))) { + return; + } + + if (includeRegistry.hasInclude(sourcePath)) { + Path relativeSourcePath = ComposerConfig.configRoot().relativize(sourcePath); + logger.info("YAML Composer: detected change in included file '{}', recompiling affected main files", + relativeSourcePath); + Set dependingMains = includeRegistry.getMainsForInclude(sourcePath); + dependingMains.forEach(mainFilePath -> processWatchEvent(Kind.MODIFY, mainFilePath)); + return; + } + + String fileName = sourcePath.getFileName().toString(); + if (!YamlComposer.isYamlFile(fileName) || YamlComposer.isIncludeFile(fileName)) { + return; + } + + includeRegistry.removeMain(sourcePath); + + Path outputPath = ComposerConfig.resolveOutputPath(sourcePath); + Path relativeSourcePath = ComposerConfig.configRoot().relativize(sourcePath); + Path relativeOutputPath = ComposerConfig.configRoot().relativize(outputPath); + if (kind == Kind.DELETE) { + try { + Files.deleteIfExists(outputPath); + logger.info("YAML Composer: DELETED generated file '{}'", relativeOutputPath); + } catch (IOException e) { + logger.warn("YAML Composer: failed to delete generated file '{}': {}", relativeOutputPath, + e.getMessage()); + } + return; + } + + try { + Object yamlObject = YamlComposer.load(sourcePath, + includePath -> includeRegistry.registerInclude(sourcePath, includePath)); + if (yamlObject == null) { + logger.warn("YAML Composer produced no output when processing '{}'", relativeSourcePath); + return; + } + + Set includePaths = includeRegistry.getIncludesForMain(sourcePath); + if (!isOutputUpToDate(sourcePath, includePaths, outputPath)) { + ComposerUtils.writeCompiledOutput(yamlObject, sourcePath, outputPath); + logger.info("YAML Composer: {} -> {}", relativeSourcePath, relativeOutputPath); + } + } catch (IOException e) { + logger.warn("YAML Composer failed to process '{}': {}", relativeSourcePath, e.getMessage()); + } + } + + private boolean isOutputUpToDate(Path sourcePath, Set includePaths, Path outputPath) { + if (!Files.exists(outputPath)) { + return false; + } + + try { + long outputMtime = Files.getLastModifiedTime(outputPath).toMillis(); + long sourceMtime = Files.getLastModifiedTime(sourcePath).toMillis(); + if (outputMtime <= sourceMtime) { + return false; + } + + for (Path includePath : includePaths) { + long includeMtime = Files.getLastModifiedTime(includePath).toMillis(); + if (outputMtime <= includeMtime) { + return false; + } + } + + return true; + } catch (IOException e) { + logger.debug("Failed to compare mtime for '{}' and '{}': {}", outputPath, sourcePath, e.getMessage()); + return false; + } + } + + private void processDirectory(Path sourceDirectory) { + try { + Files.walkFileTree(sourceDirectory, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(@Nullable Path sourcePath, @Nullable BasicFileAttributes attrs) + throws IOException { + + if (sourcePath == null || attrs == null || !attrs.isRegularFile()) { + return FileVisitResult.CONTINUE; + } + + String fileName = sourcePath.getFileName().toString(); + + if (YamlComposer.isYamlFile(fileName) && !YamlComposer.isIncludeFile(fileName)) { + processWatchEvent(Kind.CREATE, sourcePath); + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(@Nullable Path sourcePath, @Nullable IOException exc) + throws IOException { + if (sourcePath != null && exc != null) { + Path relativeSourcePath = ComposerConfig.configRoot().relativize(sourcePath); + logger.warn("Failed to process {}: {}", relativeSourcePath, exc.getMessage()); + } + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + logger.warn("Could not list YAML source files in '{}': {}", sourceDirectory, e.getMessage()); + } + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructInterpolablePlaceholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructInterpolablePlaceholder.java new file mode 100644 index 0000000000..ae3fcc37c0 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructInterpolablePlaceholder.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.constructors; + +import java.util.function.BiFunction; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.placeholders.InterpolablePlaceholder; +import org.snakeyaml.engine.v2.api.ConstructNode; +import org.snakeyaml.engine.v2.nodes.Node; + +/** + * The {@link ConstructInterpolablePlaceholder} is a generic constructor used to create + * placeholder objects for nodes whose value can be interpolated by {@link YamlComposer}. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +class ConstructInterpolablePlaceholder> implements ConstructNode { + private final BiFunction<@Nullable Object, String, T> creator; + private final ModelConstructor constructor; + + ConstructInterpolablePlaceholder(ModelConstructor constructor, BiFunction<@Nullable Object, String, T> creator) { + this.constructor = constructor; + this.creator = creator; + } + + @Override + public @Nullable Object construct(@Nullable Node node) { + if (node == null) { + return null; + } + Object value = constructor.constructByType(node); + return creator.apply(value, constructor.getLocation(node)); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructLiteral.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructLiteral.java new file mode 100644 index 0000000000..1a600cf31f --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructLiteral.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.constructors; + +import java.util.Objects; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.snakeyaml.engine.v2.api.ConstructNode; +import org.snakeyaml.engine.v2.nodes.Node; + +/** + * The {@link ConstructLiteral} is a constructor for the !literal tag. + * + * It just needs to resolve our custom tag and construct the underlying node + * as usual. + * + * We don't need a special placeholder since the tracking of substitution state + * is done in {@link ModelConstructor#constructObject(Node)}. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +class ConstructLiteral implements ConstructNode { + protected final ModelConstructor constructor; + + ConstructLiteral(ModelConstructor constructor) { + this.constructor = constructor; + } + + @Override + @NonNullByDefault({}) + public @Nullable Object construct(Node node) { + return constructor.constructByType(Objects.requireNonNull(node)); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructStr.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructStr.java new file mode 100644 index 0000000000..73a215d4b5 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructStr.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.constructors; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.snakeyaml.engine.v2.api.ConstructNode; +import org.snakeyaml.engine.v2.nodes.Node; +import org.snakeyaml.engine.v2.nodes.ScalarNode; + +/** + * The {@link ConstructStr} is the constructor used for STR tag which + * may create a {@link org.openhab.io.yamlcomposer.internal.placeholders.SubstitutionPlaceholder}, depending on the + * current + * substitution stack. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +class ConstructStr implements ConstructNode { + private final ModelConstructor constructor; + + ConstructStr(ModelConstructor constructor) { + this.constructor = constructor; + } + + @Override + @NonNullByDefault({}) + public Object construct(Node node) { + return constructor.constructScalarOrSubstitution((ScalarNode) node); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructSub.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructSub.java new file mode 100644 index 0000000000..e38696d9f2 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ConstructSub.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.constructors; + +import java.util.Objects; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.snakeyaml.engine.v2.api.ConstructNode; +import org.snakeyaml.engine.v2.nodes.Node; +import org.snakeyaml.engine.v2.nodes.Tag; + +/** + * The {@link ConstructSub} is the constructor used on the !sub tag + * to keep track of custom substitution patterns. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +class ConstructSub implements ConstructNode { + private static final String SUB_TAG_PREFIX = "sub:"; + + ModelConstructor constructor; + + ConstructSub(ModelConstructor constructor) { + this.constructor = constructor; + } + + @Override + @NonNullByDefault({}) + public Object construct(Node node) { + String patternName = extractPatternName(Objects.requireNonNull(node.getTag())); + constructor.trackPatternName(patternName); + return constructor.constructByType(node); + } + + private @Nullable String extractPatternName(Tag tag) { + String suffix = extractSubTagSuffix(tag); + if (suffix == null || suffix.isBlank()) { + return null; + } + return suffix; + } + + private @Nullable String extractSubTagSuffix(Tag tag) { + String tagValue = tag.getValue(); + + // The tag value doesn't always start with !, for example + // when using the verbatim tag syntax like ! + // the value will be "sub:pattern" without the leading "!" + // see https://yaml.org/spec/1.2.2/#verbatim-tags + if (tagValue.startsWith("!")) { + tagValue = tagValue.substring(1); + } + + if (!tagValue.startsWith(SUB_TAG_PREFIX)) { + return null; + } + + return tagValue.substring(SUB_TAG_PREFIX.length()); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ModelConstructor.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ModelConstructor.java new file mode 100644 index 0000000000..688683844b --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/constructors/ModelConstructor.java @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.constructors; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.LinkedList; +import java.util.Objects; +import java.util.Optional; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.placeholders.IfPlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.IncludePlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.InsertPlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.MergeKeyPlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.RemovePlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.ReplacePlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.SubstitutionPlaceholder; +import org.snakeyaml.engine.v2.api.ConstructNode; +import org.snakeyaml.engine.v2.api.LoadSettings; +import org.snakeyaml.engine.v2.constructor.StandardConstructor; +import org.snakeyaml.engine.v2.exceptions.Mark; +import org.snakeyaml.engine.v2.nodes.MappingNode; +import org.snakeyaml.engine.v2.nodes.Node; +import org.snakeyaml.engine.v2.nodes.ScalarNode; +import org.snakeyaml.engine.v2.nodes.SequenceNode; +import org.snakeyaml.engine.v2.nodes.Tag; + +/** + * Extends SnakeYAML Engine's {@link StandardConstructor} to add support for the + * composer's custom YAML tags and model‑transformation features. + * + *

+ * The {@code ModelConstructor} handles all extended tags, including: + *

    + *
  • !sub - variable interpolation
    • + *
  • !literal - disable interpolation for a value
    • + *
  • !if - conditional evaluation
    • + *
  • !include - include external YAML files
    • + *
  • !insert - insert templates with variable context
    • + *
  • !replace - replace parts of the model within a package
    • + *
  • !remove - remove parts of the model within a package
    • + *
+ * + * These extensions allow the composer to construct a fully evaluated + * in‑memory model before further processing or consumption. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class ModelConstructor extends StandardConstructor { + static final String SUB_TAG = "sub"; + + private static final Tag LITERAL_TAG = new Tag("!literal"); + private static final Tag IF_TAG = new Tag("!if"); + private static final Tag INCLUDE_TAG = new Tag("!include"); + private static final Tag REPLACE_TAG = new Tag("!replace"); + private static final Tag REMOVE_TAG = new Tag("!remove"); + private static final Tag INSERT_TAG = new Tag("!insert"); + + // A replacement for the built in Tag.MERGE because we want to + // do the merge key processing at the composer level + public static final Tag DEFERRED_MERGE_TAG = new Tag("!deferred-merge"); + + String sourcePath; + final Deque substitutionStack = new ArrayDeque<>(); + final Deque<@Nullable String> substitutionPatternNameStack = new LinkedList<>(); + + private final ConstructSub constructSub; + + public ModelConstructor(LoadSettings settings, String sourcePath) { + super(settings); + this.sourcePath = sourcePath; + this.substitutionStack.push(true); + this.substitutionPatternNameStack.push(null); + + this.constructSub = new ConstructSub(this); + this.tagConstructors.put(LITERAL_TAG, new ConstructLiteral(this)); + + this.tagConstructors.put(Tag.STR, new ConstructStr(this)); + + this.tagConstructors.put(IF_TAG, new ConstructInterpolablePlaceholder(this, IfPlaceholder::new)); + this.tagConstructors.put(DEFERRED_MERGE_TAG, + new ConstructInterpolablePlaceholder(this, MergeKeyPlaceholder::new)); + this.tagConstructors.put(INCLUDE_TAG, + new ConstructInterpolablePlaceholder(this, IncludePlaceholder::new)); + this.tagConstructors.put(INSERT_TAG, + new ConstructInterpolablePlaceholder(this, InsertPlaceholder::new)); + this.tagConstructors.put(REPLACE_TAG, + new ConstructInterpolablePlaceholder(this, ReplacePlaceholder::new)); + this.tagConstructors.put(REMOVE_TAG, + new ConstructInterpolablePlaceholder(this, RemovePlaceholder::new)); + } + + @Override + @NonNullByDefault({}) + @SuppressWarnings("null") + protected Optional findConstructorFor(Node node) { + if (isSubstitutionTag(node.getTag())) { + return Optional.of(constructSub); + } + return super.findConstructorFor(node); + } + + /** + * Gets a string representation of the node's location for logging purposes. + * + * @param node the YAML node to get the location of + * @return a string describing the source location of the node, including file path and line/column if available + */ + String getLocation(Node node) { + String location = ""; + Mark startMark = node.getStartMark().orElse(null); + if (startMark != null) { + location = ":%d:%d".formatted(startMark.getLine() + 1, startMark.getColumn() + 1); + } + return this.sourcePath + location; + } + + /** + * Default construction method that routes to the appropriate construct method + * based on the node type. + * + * Use this instead of constructObject() to avoid an infinite recursion when + * constructing a node on a custom tag. + * + * @param node the node to construct + * @return the constructed object + */ + protected @Nullable Object constructByType(Node node) { + return switch (node) { + case MappingNode mappingNode -> constructMapping(mappingNode); + case SequenceNode sequenceNode -> constructSequence(sequenceNode); + case ScalarNode scalarNode -> constructScalarOrSubstitution(scalarNode); + default -> constructObject(node); + }; + } + + /** + * Construct a scalar node, potentially as a SubstitutionPlaceholder + * if the current substitution state is enabled. + * + * @param scalarNode the scalar node to construct + * @return + */ + @SuppressWarnings("null") // The stacks and SnakeYAML methods shouldn't return null + protected @Nullable Object constructScalarOrSubstitution(ScalarNode scalarNode) { + Tag tag = scalarNode.getTag(); + String value = constructScalar(scalarNode); + boolean enabled = substitutionStack.peek(); + if (enabled || isSubstitutionTag(tag)) { + String patternName = substitutionPatternNameStack.peek(); + String location = getLocation(scalarNode); + return new SubstitutionPlaceholder(value, patternName, location); + } + return value; + } + + /** + * Intercept constructObject to keep track of the current substitution state. + * + * @param node the node to construct + * @return the constructed object + */ + @Override + @NonNullByDefault({}) + protected @Nullable Object constructObject(Node node) { + Tag tag = Objects.requireNonNull(node.getTag()); + boolean parent = Objects.requireNonNull(substitutionStack.peek()); + boolean enabled = resolveSubstitution(tag, parent); + substitutionStack.push(enabled); + substitutionPatternNameStack.push(substitutionPatternNameStack.peek()); + try { + return super.constructObject(node); + } finally { + substitutionPatternNameStack.pop(); + substitutionStack.pop(); + } + } + + private static boolean resolveSubstitution(Tag tag, boolean parent) { + if (LITERAL_TAG.equals(tag)) { + return false; + } + + if (isSubstitutionTag(tag)) { + return true; + } + return parent; + } + + static boolean isSubstitutionTag(Tag tag) { + String value = tag.getValue(); + if (value.startsWith("!")) { + value = value.substring(1); + } + return value.startsWith(SUB_TAG); + } + + protected void trackPatternName(@Nullable String patternName) { + substitutionPatternNameStack.pop(); + substitutionPatternNameStack.push(patternName); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/PackageProcessor.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/PackageProcessor.java new file mode 100644 index 0000000000..f07be74652 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/PackageProcessor.java @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.core; + +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.BufferedLogger; +import org.openhab.io.yamlcomposer.internal.placeholders.ReplacePlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.SubstitutionPlaceholder; + +/** + * Processor for handling 'packages' in YAML models. + * + * Merges package definitions into the main data structure, injecting the package ID + * into any included or inserted fragments. + * + * This is slightly different to the other processors as it operates on the overall + * data structure rather than merely performing resolutions of a particular placeholder. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class PackageProcessor { + private static final String PACKAGES_KEY = "packages"; + private static final String PACKAGE_ID_VAR = "package_id"; + + private final BufferedLogger logger; + private final Path absolutePath; + private final Path relativePath; + private final SourceLocator sourceLocator; + private final RecursiveTransformer recursiveTransformer; + + public PackageProcessor(BufferedLogger logger, Path absolutePath, Path relativePath, SourceLocator sourceLocator, + RecursiveTransformer recursiveTransformer) { + this.logger = logger; + this.absolutePath = absolutePath; + this.relativePath = relativePath; + this.sourceLocator = sourceLocator; + this.recursiveTransformer = recursiveTransformer; + } + + /** + * Convenience wrapper that accepts the raw 'packages' section value and + * applies merging if it's a map, otherwise logs the same warning used by + * the composer. + * + * @param yamlMap the root YAML map to merge packages into + * @param packagesObj the raw 'packages' section value to process and merge, or null if not present + */ + public void mergePackages(Map yamlMap, @Nullable Object packagesObj) { + if (packagesObj instanceof Map packagesMap) { + mergePackages(yamlMap, packagesMap); + logger.debug("Merged packages into data in {}: {}", absolutePath, yamlMap); + } else if (packagesObj != null) { + var position = sourceLocator.findPosition(PACKAGES_KEY); + logger.warn("{}:{} The 'packages' section is not a map", relativePath, position); + } + } + + /** + * Deep merge packages map into the main data map + */ + private void mergePackages(Map mainData, Map packages) { + packages.forEach((pkgKey, pkg) -> { + Object pkgKeyObj = recursiveTransformer.transform(pkgKey); + if (pkgKeyObj == null) { + var position = sourceLocator.findPosition(PACKAGES_KEY); + logger.warn("{}:{} package key resolved to null; skipping package entry", relativePath, position); + return; + } + String packageId = String.valueOf(pkgKeyObj); + + // Inject `package_id` into the package context for use in + // !include and !insert within the package definition + Map packageIdMap = Map.of(PACKAGE_ID_VAR, packageId); + RecursiveTransformer packageTransformer = recursiveTransformer.withOverrideVariables(packageIdMap); + Object resolvedPkg = packageTransformer.transform(pkg, ProcessingPhase.STANDARD); + + resolvedPkg = stripEmptyMapsAndLists(resolvedPkg); + if (!(resolvedPkg instanceof Map packageMap)) { + var position = sourceLocator.findPosition(PACKAGES_KEY, packageId); + logger.warn("{}:{} package '{}' resolved to {} instead of a Map", relativePath, position, packageId, + resolvedPkg == null ? "null" : resolvedPkg.getClass().getSimpleName()); + return; + } + + logger.debug("Merging package '{}' {} into main data: {}", packageId, packageMap, mainData); + mergeElements(packageId, mainData, packageMap); + }); + } + + private void mergeElements(String packageId, Map mainData, Map packageData) { + packageData.forEach((packageKey, packageValue) -> { + if (mainData.containsKey(packageKey)) { + Object mainValue = mainData.get(packageKey); + if (mainValue instanceof ReplacePlaceholder || mainValue instanceof SubstitutionPlaceholder) { + return; // Ignore the package value - we'll process these later + } + if (mainValue instanceof Map mainValueMap) { + if (packageValue instanceof Map packageValueMap) { + mergeElements(packageId, mainValueMap, packageValueMap); + @SuppressWarnings("unchecked") + Map<@Nullable Object, Object> rawMainData = (Map<@Nullable Object, Object>) mainData; + rawMainData.put(packageKey, mainValueMap); + return; + } else { + // Type mismatch - keep main value + var position = sourceLocator.findPosition(PACKAGES_KEY, packageId); + logger.warn( + "{}:{} Type mismatch when merging package ID '{}' for key '{}': main is Map, package is {}; keeping main value", + relativePath, position, packageId, packageKey, + packageValue == null ? "null" : packageValue.getClass().getSimpleName()); + return; + } + } + if (mainValue instanceof List mainValueList) { + if (packageValue instanceof List pkgValueList) { + @SuppressWarnings("unchecked") + Map<@Nullable Object, Object> rawMainData = (Map<@Nullable Object, Object>) mainData; + rawMainData.put(packageKey, + Stream.concat(pkgValueList.stream(), mainValueList.stream()).distinct().toList()); + return; + } else { + // Type mismatch - keep main value + var position = sourceLocator.findPosition(PACKAGES_KEY, packageId); + logger.warn( + "{}:{} Type mismatch when merging package ID '{}' for key '{}': main is List, package is {}; keeping main value", + relativePath, position, packageId, packageKey, + packageValue == null ? "null" : packageValue.getClass().getSimpleName()); + return; + } + } + // For non-map/non-list values, keep the main value ignoring the package value + } else { + @SuppressWarnings("unchecked") + Map<@Nullable Object, @Nullable Object> rawMainData = (Map<@Nullable Object, @Nullable Object>) mainData; + rawMainData.put(packageKey, packageValue); + } + }); + } + + private static @Nullable Object stripEmptyMapsAndLists(@Nullable Object data) { + if (data == null || data instanceof String s && s.isBlank()) { + return null; + } + if (data instanceof Map map) { + var result = new LinkedHashMap(); + for (Map.Entry e : map.entrySet()) { + Object key = e.getKey(); + Object value = stripEmptyMapsAndLists(e.getValue()); + if (value != null) { + result.put(key, value); + } + } + return result.isEmpty() ? null : result; + } else if (data instanceof List list) { + var result = new java.util.ArrayList(list.size()); + for (Object item : list) { + Object value = stripEmptyMapsAndLists(item); + if (value != null) { + result.add(value); + } + } + return result.isEmpty() ? null : result; + } + return data; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/ProcessingPhase.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/ProcessingPhase.java new file mode 100644 index 0000000000..ebb99f9d1f --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/ProcessingPhase.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.core; + +import java.util.Set; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.openhab.io.yamlcomposer.internal.placeholders.IfPlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.IncludePlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.InsertPlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.Placeholder; +import org.openhab.io.yamlcomposer.internal.placeholders.RemovePlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.ReplacePlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.SubstitutionPlaceholder; + +/** + * Defines the different processing phases for the YAML composer. + * + * @author Jimmy Tanagra - Initial Contribution + */ +@NonNullByDefault +public class ProcessingPhase { + public static final Set> SUBSTITUTION = // + Set.of(SubstitutionPlaceholder.class); + public static final Set> INCLUDES = // + Set.of(IncludePlaceholder.class); + public static final Set> STANDARD = // + Set.of(SubstitutionPlaceholder.class, IfPlaceholder.class, IncludePlaceholder.class, + InsertPlaceholder.class); + public static final Set> PACKAGE_OVERRIDES = // + Set.of(RemovePlaceholder.class, ReplacePlaceholder.class); + + private ProcessingPhase() { + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/RecursiveTransformer.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/RecursiveTransformer.java new file mode 100644 index 0000000000..b1b1919a10 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/RecursiveTransformer.java @@ -0,0 +1,361 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.core; + +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.placeholders.IfPlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.InterpolablePlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.MergeKeyPlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.Placeholder; +import org.openhab.io.yamlcomposer.internal.placeholders.SubstitutionPlaceholder; +import org.openhab.io.yamlcomposer.internal.processors.PlaceholderProcessor; + +/** + * The {@link RecursiveTransformer} traverses a YAML data tree, applies merge keys logic, + * and transforms placeholders into the final values by invoking registered handlers + * for their respective types. + * + * It holds a variables map that can be overridden for nested transformations, + * allowing for context-specific variable values during processing. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class RecursiveTransformer { + + private final Map, PlaceholderProcessor> handlers = new LinkedHashMap<>(); + private final Map variables; + + public RecursiveTransformer(Map variables) { + this.variables = variables; + } + + public Map getVariables() { + return variables; + } + + /** + * Creates a new RecursiveTransformer with the same handlers but a new variables map that + * includes the given overrides. + * + * The new combined variables map will be used for all placeholder processing within + * the nested structure, allowing for isolated modifications without impacting the original context. + * + * @param overrideVariables additional variables to include in the new transformer's context, + * which will override any existing variables with the same keys + * @return a new RecursiveTransformer instance with the combined variables and the same handlers + */ + public RecursiveTransformer withOverrideVariables(Map overrideVariables) { + Map combinedVariables = new HashMap<>(variables); + combinedVariables.putAll(overrideVariables); + RecursiveTransformer copy = new RecursiveTransformer(combinedVariables); + copy.handlers.putAll(this.handlers); + return copy; + } + + /** + * Registers a handler for a specific placeholder type. + * + * @param handler the processor that can transform the placeholder + */ + public void register(PlaceholderProcessor handler) { + handlers.put(handler.getPlaceholderType(), handler); + } + + /** + * Default: transforms the entire tree using all registered handlers. + * Placeholders are transformed if their class matches any registered handler. + * + * This is the most common usage, but the other overloads allow for more control + * and optimization by restricting which handlers are applied. + * + * @param data the YAML data tree to transform + * @return the transformed data tree with placeholders transformed + */ + public @Nullable Object transform(@Nullable Object data) { + return transformWithVisited(data, handlers.keySet()); + } + + /** + * Transforms the data tree but only applies handlers for the specified placeholder classes. + * + *

+ * Use this when you want to restrict transformation to a subset of placeholder + * classes. + * + * @param data the YAML data tree to transform + * @param allowedTypes the set of placeholder classes to transform + * @return the transformed data tree with the given placeholders transformed + */ + public @Nullable Object transform(@Nullable Object data, Set> allowedTypes) { + return transformWithVisited(data, allowedTypes); + } + + /** + * Convenience overload for transforming a Map container. + * + *

+ * Transforms the given map and transforms any placeholders found within keys and values. + * This is equivalent to calling {@link #transform(Object, Set)} with the full set of + * registered placeholder handler types. + * + * @param data the map to transform + * @return the transformed map + */ + public Map transform(Map data) { + return transform(data, handlers.keySet()); + } + + /** + * Transforms the given map but only applies handlers for the specified placeholder classes. + * + *

+ * Keys and values are transformed recursively. If the overall transformed result is not a + * {@link Map} (for example, if a placeholder handler returned a non-container), an + * {@link IllegalStateException} is thrown. + * + * @param data the map to transform + * @param allowedTypes the set of placeholder classes to transform + * @return the transformed map + * @throws IllegalStateException if the transformed result is not a Map + */ + @SuppressWarnings("unchecked") + public Map transform(Map data, Set> allowedTypes) { + Object transformed = transformWithVisited(data, allowedTypes); + if (transformed instanceof Map map) { + return (Map) map; + } + throw new IllegalStateException("Expected transformed result to be a Map but was: " + + (transformed == null ? "null" : transformed.getClass())); + } + + /** + * Central helper to start transformation with a fresh visited map. + */ + private @Nullable Object transformWithVisited(@Nullable Object data, + Set> allowedTypes) { + return transformInternal(data, allowedTypes, new IdentityHashMap<>()); + } + + /** + * The actual tree traversal. + */ + private @Nullable Object transformInternal(@Nullable Object data, Set> allowedTypes, + IdentityHashMap visited) { + + if (data == null) { + return null; + } + + Class clazz = data.getClass(); + + // Handle cyclic references for containers: if we've already started transforming + // this container, return the placeholder/result to avoid infinite recursion. + if ((data instanceof Map || data instanceof List)) { + if (visited.containsKey(data)) { + return visited.get(data); + } + } + + // Resolve placeholder value (arguments) first before transforming the placeholder itself + // So that e.g. !include ${filename} gets the real argument value to transform + if (data instanceof InterpolablePlaceholder interpolable) { + Object transformedValue; + + if (interpolable.eagerArgumentProcessing()) { + // Eagerly transform arguments using all registered handlers + transformedValue = transform(interpolable.value()); + } else { + // Only perform substitutions in arguments + // for !if conditions, don't transform placeholders (e.g. !include) in the unselected branch + transformedValue = transform(interpolable.value(), ProcessingPhase.SUBSTITUTION); + } + data = interpolable.withValue(transformedValue); + } + + if (data instanceof Placeholder placeholder && allowedTypes.contains(clazz)) { + // Use the override callback if provided, otherwise look up in registry + PlaceholderProcessor handler = handlers.get(clazz); + + if (handler != null) { + // Execute and recurse + Object result = invokeHandler(handler, placeholder); + return transformInternal(result, allowedTypes, visited); + } + } + + if (data instanceof Map map) { + return resolveMap(map, allowedTypes, visited); + } + + if (data instanceof List list) { + return resolveList(list, allowedTypes, visited); + } + + return data; + } + + @SuppressWarnings("unchecked") + private @Nullable Object invokeHandler(PlaceholderProcessor handler, + Placeholder placeholder) { + return ((PlaceholderProcessor) handler).process((T) placeholder, this); + } + + /** + * Resolves a map by transforming its keys and values, applying placeholder handlers as needed, + * and handling special cases like merge keys and removal signals. + * + * @param rawMap the original map to transform + * @param allowedTypes the set of placeholder classes to transform + * @return the transformed map with placeholders transformed, or the original map if no changes were made + */ + private Object resolveMap(Map rawMap, Set> allowedTypes, + IdentityHashMap visited) { + // Always create a new map for transformed results to simplify cycle handling + @SuppressWarnings("unchecked") + Map map = (Map) rawMap; + + Map result = new LinkedHashMap<>(map.size()); + // Register in visited before transforming entries to handle self-references + visited.put(rawMap, result); + + List> mergeEntries = new ArrayList<>(); + + for (Map.Entry entry : map.entrySet()) { + Object oldKey = entry.getKey(); + Object oldVal = entry.getValue(); + + Object newKey = transformInternal(oldKey, allowedTypes, visited); + Object newVal = transformInternal(oldVal, allowedTypes, visited); + + // Dropping null keys or removal signals + if (shouldRemoveEntry(newKey, newVal, oldVal)) { + continue; + } + + newKey = Objects.requireNonNull(newKey); // null keys should have been filtered out in shouldRemoveEntry + + if (newKey instanceof MergeKeyPlaceholder mkp) { + mergeEntries.add(new AbstractMap.SimpleEntry<>(mkp, newVal)); + continue; + } + + if ("<<".equals(newKey)) { + mergeEntries.add(new AbstractMap.SimpleEntry<>(newKey, newVal)); + continue; + } + + result.put(newKey, newVal); + } + + resolveMergeKeys(result, allowedTypes, mergeEntries, visited); + + return result; + } + + private boolean shouldRemoveEntry(@Nullable Object newKey, @Nullable Object newVal, @Nullable Object oldVal) { + return newKey == null // + || newKey == RemovalSignal.REMOVE // + || newVal == RemovalSignal.REMOVE // + || (newVal == null && oldVal != null); + } + + public void resolveMergeKeys(Map rawMap) { + resolveMergeKeys(rawMap, Set.of()); + } + + public void resolveMergeKeys(Map rawMap, Set> allowedTypes) { + @SuppressWarnings("unchecked") + Map map = (Map) rawMap; + + // First: handle YAML merge keys (<<: ...). We collect merge-key entries, transform their + // values fully, merge into this container, and remove the merge-key entries before + // performing the normal per-entry transformation below. + List> mergeEntries = new ArrayList<>(); + for (Map.Entry entry : map.entrySet()) { + if (entry.getKey() instanceof MergeKeyPlaceholder || "<<".equals(entry.getKey())) { + mergeEntries.add(new AbstractMap.SimpleEntry<>(entry.getKey(), entry.getValue())); + } + } + + resolveMergeKeys(map, allowedTypes, mergeEntries, new IdentityHashMap<>()); + } + + @SuppressWarnings({ "unchecked" }) + private void resolveMergeKeys(Map rawMap, Set> allowedTypes, + List> mergeEntries, IdentityHashMap visited) { + + Map map = (Map) rawMap; + + if (!mergeEntries.isEmpty()) { + for (var mergeEntry : mergeEntries) { + @Nullable + Object rawVal = mergeEntry.getValue(); + @Nullable + Object transformedVal = transformInternal(rawVal, allowedTypes, visited); + + if (transformedVal instanceof Map fromMap) { + mergeMap((Map) fromMap, map); + } else if (transformedVal instanceof List list) { + for (Object item : list) { + if (item instanceof Map m) { + mergeMap((Map) m, map); + } + } + } else if (transformedVal == null) { + if (rawVal instanceof SubstitutionPlaceholder || rawVal instanceof IfPlaceholder) { + // nothing to merge + } + } + + map.remove(mergeEntry.getKey()); + } + } + } + + private void mergeMap(Map from, Map to) { + for (Map.Entry entry : from.entrySet()) { + Object key = entry.getKey(); + if (!to.containsKey(key)) { + to.put(key, entry.getValue()); + } + } + } + + private Object resolveList(List list, Set> allowedTypes, + IdentityHashMap visited) { + // Always produce a new list and register it to handle cycles + List<@Nullable Object> result = new ArrayList<>(list.size()); + visited.put(list, result); + + for (Object oldItem : list) { + Object newItem = transformInternal(oldItem, allowedTypes, visited); + if (newItem != RemovalSignal.REMOVE && newItem != null) { + result.add(newItem); + } + } + + return result; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/RemovalSignal.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/RemovalSignal.java new file mode 100644 index 0000000000..0ffbddf7da --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/RemovalSignal.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.core; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * A signal returned by {@link org.openhab.io.yamlcomposer.internal.processors.PlaceholderProcessor}s to indicate that + * the entry + * currently being processed should be intentionally removed from its + * parent container (Map or List). + *

+ * This enum provides an explicit way to signal removal, distinguishing it from + * error-based removal that occurs when a processor returns {@code null}. + * + *

Usage Guidelines

+ *

+ * Use {@code RemovalSignal.REMOVE} when: + *

    + *
  • The placeholder's purpose is to explicitly remove an entry (e.g., {@code !remove} directive)
    • + *
  • The removal is intentional and by design, not due to an error condition
    • + *
  • You want to make the removal intent clear in the code
    • + *
+ * + * Use {@code null} when: + *
    + *
  • Processing failed due to an error (missing parameter, invalid input, resource not found)
    • + *
  • You've logged a warning describing the error before returning {@code null}
    • + *
  • The entry should be removed as error recovery, not as intended functionality
    • + *
+ * + *

Technical Details

+ *

+ * The {@link RecursiveTransformer} treats both {@code RemovalSignal.REMOVE} and {@code null} + * identically in terms of behavior (both cause entry removal), but the semantic distinction + * helps developers understand whether removal was intentional or due to error recovery. + * + * @author Jimmy Tanagra - Initial contribution + * @see org.openhab.io.yamlcomposer.internal.processors.PlaceholderProcessor#process + */ +@NonNullByDefault +public enum RemovalSignal { + /** + * Signals that the current entry should be intentionally removed from its parent container. + *

+ * This is the recommended way to explicitly indicate removal as part of normal processing, + * as opposed to returning {@code null} which should be reserved for error cases. + */ + REMOVE; +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/SourceLocator.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/SourceLocator.java new file mode 100644 index 0000000000..0af23fd24a --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/SourceLocator.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.core; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Scanner; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * Helper to find positions in YAML source for logging. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class SourceLocator { + private final byte[] yamlBytes; + + public SourceLocator(byte[] yamlBytes) { + this.yamlBytes = yamlBytes; + } + + public record FilePosition(int line, int column) { + @Override + public String toString() { + if (line < 0) { + return ""; + } + return line + ":" + column; + } + + public static FilePosition empty() { + return new FilePosition(-1, -1); + } + } + + /** + * Find the position of a given key in the YAML file. + * + * @param keys the sequence of keys representing the path in the YAML structure + * @return the Position of the last key in the sequence, or (-1, -1) if not + * found + */ + public FilePosition findPosition(String... keys) { + if (keys.length == 0) { + return FilePosition.empty(); + } + + try (Scanner scanner = new Scanner(new ByteArrayInputStream(yamlBytes), StandardCharsets.UTF_8)) { + int lineNumber = 1; + int keyIndex = 0; + + while (scanner.hasNextLine()) { + String line = scanner.nextLine(); + int lineOffset = 0; // Tracks our horizontal position in the current line + String searchArea = line; + boolean foundInLine; + + do { + foundInLine = false; + String targetKey = keys[keyIndex] + ":"; + int matchIndex = searchArea.indexOf(targetKey); + + if (matchIndex != -1) { + // Calculate the column: + // segments already skipped + position in current segment + length of key + int columnAtEndOfKey = lineOffset + matchIndex + targetKey.length(); + + if (keyIndex == keys.length - 1) { + return new FilePosition(lineNumber, columnAtEndOfKey + 1); // +1 for 1-based indexing + } + + // Prepare for next key on the same line + lineOffset += matchIndex + targetKey.length(); + searchArea = searchArea.substring(matchIndex + targetKey.length()); + keyIndex++; + foundInLine = true; + } + } while (foundInLine && keyIndex < keys.length); + + lineNumber++; + } + } + return FilePosition.empty(); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/TemplateLoader.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/TemplateLoader.java new file mode 100644 index 0000000000..d83f56b269 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/TemplateLoader.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.core; + +import java.nio.file.Path; +import java.util.Map; +import java.util.Objects; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.BufferedLogger; +import org.openhab.io.yamlcomposer.internal.ComposerConfig; + +/** + * The {@link TemplateLoader} is responsible for extracting templates from the YAML model and storing them in the + * composer context for later use. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class TemplateLoader { + private final BufferedLogger logger; + private final Path relativePath; + private final Map templates; + private final RecursiveTransformer recursiveTransformer; + private final SourceLocator locator; + + public TemplateLoader(BufferedLogger logger, Path relativePath, Map templates, + RecursiveTransformer recursiveTransformer, SourceLocator locator) { + this.logger = logger; + this.relativePath = relativePath; + this.templates = templates; + this.recursiveTransformer = recursiveTransformer; + this.locator = locator; + } + + /** + * Extracts templates from the given map and stores them into the templates map. + * + * @param templatesSection the section of the YAML model containing templates + */ + public void extractTemplates(@Nullable Object templatesSection) { + if (templatesSection instanceof java.util.Map templatesMap) { + templatesMap.keySet().removeIf(Objects::isNull); + recursiveTransformer.resolveMergeKeys(templatesMap, ProcessingPhase.STANDARD); + templatesMap.forEach((key, value) -> { + // Only resolve the key so we can look up the template name in the templates map. + // The value substitutions must NOT be resolved here! + // They will be resolved at insertion-time using insertion context instead of the main variables. + Object resolvedKey = recursiveTransformer.transform(key, ProcessingPhase.SUBSTITUTION); + if (resolvedKey != null) { + templates.put(resolvedKey, value); + } + }); + } else if (templatesSection != null) { + var position = locator.findPosition(ComposerConfig.TEMPLATES_KEY); + logger.warn("{}:{} 'templates' is not a map", relativePath, position); + } + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/VariableLoader.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/VariableLoader.java new file mode 100644 index 0000000000..f85a5467d1 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/core/VariableLoader.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.core; + +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.BufferedLogger; +import org.openhab.io.yamlcomposer.internal.ComposerConfig; +import org.openhab.io.yamlcomposer.internal.placeholders.IncludePlaceholder; +import org.openhab.io.yamlcomposer.internal.placeholders.MergeKeyPlaceholder; + +/** + * The {@link VariableLoader} is responsible for extracting variable definitions from the YAML model and storing + * them in the composer context for later use in substitutions. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class VariableLoader { + private final Map variables; + private final Path absolutePath; + private final RecursiveTransformer recursiveTransformer; + private final BufferedLogger logger; + + public VariableLoader(Map variables, Path absolutePath, + RecursiveTransformer recursiveTransformer, BufferedLogger logger) { + this.variables = variables; + this.absolutePath = absolutePath; + this.recursiveTransformer = recursiveTransformer; + this.logger = logger; + } + + /** + * Add special file-related variables + * + * These are added early so they're available in variable definitions during the + * first pass + * Special variables will override any user-defined variables with the same name + */ + public void setSpecialVariables() { + Path fileNamePath = absolutePath.getFileName(); + String fullFileName = fileNamePath != null ? fileNamePath.toString() : ""; + int dotIndex = fullFileName.lastIndexOf("."); + String fileName = fullFileName; + String fileExtension = ""; + if (dotIndex > 0) { + fileName = fullFileName.substring(0, dotIndex); + fileExtension = fullFileName.substring(dotIndex + 1); + } + var parentPath = absolutePath.getParent(); + String directory = parentPath != null ? parentPath.toString() : ""; + Map vars = variables; + + vars.put("OPENHAB_CONF", ComposerConfig.configRoot().toString()); + vars.put("OPENHAB_USERDATA", ComposerConfig.userDataRoot().toString()); + vars.put("__FILE__", absolutePath.toString()); + vars.put("__FILE_NAME__", fileName); + vars.put("__FILE_EXT__", fileExtension); + vars.put("__DIRECTORY__", directory); + vars.put("__DIR__", directory); + } + + /** + * Extracts variables from the given map and stores them into the context's variable map. + * + * Since variables can reference previously defined variables, we perform incremental resolution + * while iterating through the variable definitions. + * + * @param variablesSection the section of the YAML file containing variable definitions, can be null + * @param locator the source locator for logging purposes + * @see ComposerConfig#VARIABLES_KEY + */ + public void extractVariables(@Nullable Object variablesSection, SourceLocator locator) { + Map existingVariables = variables; + + if (variablesSection instanceof Map variablesMap) { + Map mergeKeys = new LinkedHashMap<>(); + + variablesMap.forEach((key, value) -> { + if (key instanceof MergeKeyPlaceholder) { + mergeKeys.put(key, value); + return; + } + + Object keyObj = recursiveTransformer.transform(key); + if (keyObj == null) { + return; + } + + String keyStr = String.valueOf(keyObj); + if (!existingVariables.containsKey(keyStr)) { + Object resolvedValue = recursiveTransformer.transform(value); + existingVariables.put(keyStr, resolvedValue); + } + }); + + if (!mergeKeys.isEmpty()) { + Map mergedVars = new LinkedHashMap<>(existingVariables); + Map processedMergeKeys = recursiveTransformer.transform(mergeKeys); + mergedVars.putAll(processedMergeKeys); + recursiveTransformer.resolveMergeKeys(mergedVars, ProcessingPhase.STANDARD); + existingVariables.clear(); + mergedVars.forEach((k, v) -> existingVariables.put(String.valueOf(k), v)); + } + } else if (variablesSection instanceof IncludePlaceholder includePlaceholder) { + Object includedData = recursiveTransformer.transform(includePlaceholder, ProcessingPhase.INCLUDES); + extractVariables(includedData, locator); + } else if (variablesSection != null) { + var position = locator.findPosition(ComposerConfig.VARIABLES_KEY); + Path relativePath = ComposerConfig.configRoot().relativize(absolutePath); + logger.warn("{}:{} 'variables' is not a map", relativePath, position); + } + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/expression/ExpressionEvaluator.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/expression/ExpressionEvaluator.java new file mode 100644 index 0000000000..a037a4e08e --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/expression/ExpressionEvaluator.java @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.expression; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.LogSession; +import org.openhab.io.yamlcomposer.internal.expression.filters.DigFilter; +import org.openhab.io.yamlcomposer.internal.expression.filters.LabelFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.hubspot.jinjava.Jinjava; +import com.hubspot.jinjava.JinjavaConfig; +import com.hubspot.jinjava.features.FeatureConfig; +import com.hubspot.jinjava.features.FeatureStrategies; +import com.hubspot.jinjava.interpret.Context; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.interpret.TemplateError; +import com.hubspot.jinjava.util.ObjectTruthValue; + +/** + * Wrapper around Jinjava template engine for rendering ${...} expressions. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class ExpressionEvaluator { + + private static final JinjavaConfig CONFIG; + private static final Jinjava JINJAVA; + private static final Logger LOGGER = LoggerFactory.getLogger(ExpressionEvaluator.class); + + static { + @SuppressWarnings("null") + JinjavaConfig config = JinjavaConfig.newBuilder() // + .withFeatureConfig(FeatureConfig.newBuilder() + .add(JinjavaInterpreter.OUTPUT_UNDEFINED_VARIABLES_ERROR, FeatureStrategies.ACTIVE).build()) + .withFailOnUnknownTokens(false) // + .withMaxRenderDepth(1) // + .withMaxMacroRecursionDepth(0) // We don't use macros; disallow recursion + .withEnableRecursiveMacroCalls(false) // + .build(); + CONFIG = config; + JINJAVA = new Jinjava(CONFIG); + Context context = JINJAVA.getGlobalContext(); + context.registerFilter(new LabelFilter()); + context.registerFilter(new DigFilter()); + } + + /** + * Evaluate a Jinjava expression and return the raw object result (no string coercion). + * + * @param expression the expression content without delimiters (e.g., "user.profile") + * @param variables the variable context + * @return the evaluated object in its native type + */ + public static @Nullable Object renderObject(String expression, Map variables, + LogSession logSession, String sourceLocation) { + @SuppressWarnings("null") + Context context = new Context(JINJAVA.getGlobalContext(), variables); + context.setDynamicVariableResolver(varName -> dynamicVariableResolver(varName, variables)); + JinjavaInterpreter interpreter = new JinjavaInterpreter(JINJAVA, context, CONFIG); + + Object result = interpreter.resolveELExpression(expression, 0); + + @SuppressWarnings("null") + List errors = interpreter.getErrorsCopy(); + if (!errors.isEmpty()) { + List messages = errors.stream().map((e) -> { + String msg = Objects.requireNonNullElse(e.getMessage(), ""); + if (e.getItem() == TemplateError.ErrorItem.TOKEN) { + String missingVarName = extractVariableName(e); + if (missingVarName != null && variables.containsKey(missingVarName)) { + // Jinjava emits a TOKEN error when a referenced variable resolves to null. + // If the variable key exists in our variables map (even if its value is null), + // treat it as intentionally defined and suppress an "undefined variable" suggestion. + return null; + } + String suggestion = findClosestVariableName(missingVarName, variables); + if (suggestion != null) { + msg += " (Did you mean '" + suggestion + "'?)"; + } + } + return msg; + }).filter(Objects::nonNull).toList(); + + if (!messages.isEmpty()) { + String combinedMessage = (messages.size() == 1) // + ? " " + messages.getFirst() // + : "\n- " + String.join("\n- ", messages); + + String logMsg = String.format("%s Error evaluating expression '%s':%s", sourceLocation, expression, + combinedMessage); + + logSession.trackWarning(LOGGER, logMsg); + } + } + return normalizeType(result); + } + + /** + * Determines the truthiness of a value according to Jinjava rules. + * + * @param value the value to evaluate + * @return true if the value is considered truthy, false otherwise + */ + public static boolean isTruthy(@Nullable Object value) { + return ObjectTruthValue.evaluate(value); + } + + /** + * Dynamic variable resolver for Jinjava to provide special variables like "VARS" and "ENV". + * + * @param varName the name of the variable being resolved + * @param context the current variable context + * @return the value of the special variable, or null if it's not a special variable + */ + private static @Nullable Object dynamicVariableResolver(@Nullable String varName, + Map context) { + if ("VARS".equals(varName)) { + return context; + } + + if ("ENV".equals(varName)) { + return System.getenv(); + } + return null; + } + + /** + * Normalize types returned by Jinjava to match SnakeYAML's + * + * Jinjava tends to return Long for all integer numbers. + * SnakeYAML, however, uses Integer for small integers. + * Normalize accordingly to avoid type mismatches. + * + * Example: + * + * ```yaml + * native: 1 # returns a Java Integer + * jinja: ${ 1 } # before normalization returns a Java Long + * ``` + * + * @param obj the object to normalize + * @return the normalized object + */ + private static @Nullable Object normalizeType(@Nullable Object obj) { + if (obj instanceof Long longValue) { + // Check if the value fits in an Integer + if (longValue >= Integer.MIN_VALUE && longValue <= Integer.MAX_VALUE) { + return Math.toIntExact(longValue); + } + } + return obj; + } + + private static @Nullable String extractVariableName(TemplateError error) { + @SuppressWarnings("null") + Map categoryErrors = error.getCategoryErrors(); + String variableName = null; + if (categoryErrors != null && categoryErrors.containsKey("variable")) { + variableName = categoryErrors.get("variable"); + } + + if (variableName == null) { + String message = Objects.requireNonNullElse(error.getMessage(), ""); + variableName = message.replaceAll(".*'([^']+)'.*", "$1"); + } + return variableName; + } + + /** + * Finds the most similar variable name using a standalone Levenshtein implementation. + */ + private static @Nullable String findClosestVariableName(@Nullable String missingVar, + Map variables) { + if (missingVar == null) { + return null; + } + String bestMatch = null; + int maxDistance = 3; // Threshold: only suggest if 1 or 2 edits away + + for (String existingVar : variables.keySet()) { + int distance = levenshteinDistance(missingVar, existingVar, maxDistance); + if (distance < maxDistance) { + maxDistance = distance; + bestMatch = existingVar; + } + } + return bestMatch; + } + + /** + * Calculate the Levenshtein distance between two strings with an early exit if the distance exceeds the threshold. + * + * @param s1 The first string + * @param s2 The second string + * @param threshold The maximum distance to calculate before exiting early + * @return the Levenshtein distance, or Integer.MAX_VALUE if the distance exceeds the threshold + */ + private static int levenshteinDistance(@Nullable String s1, @Nullable String s2, int threshold) { + if (s1 == null || s2 == null) { + return Integer.MAX_VALUE; + } + + int len1 = s1.length(); + int len2 = s2.length(); + + // If length difference is greater than our threshold, skip expensive math + if (Math.abs(len1 - len2) >= threshold) { + return Integer.MAX_VALUE; + } + + int[] costs = new int[len2 + 1]; + for (int i = 0; i <= len1; i++) { + int lastValue = i; + for (int j = 0; j <= len2; j++) { + if (i == 0) { + costs[j] = j; + } else if (j > 0) { + int newValue = costs[j - 1]; + if (s1.charAt(i - 1) != s2.charAt(j - 1)) { + newValue = Math.min(Math.min(newValue, lastValue), costs[j]) + 1; + } + costs[j - 1] = lastValue; + lastValue = newValue; + } + } + if (i > 0) { + costs[len2] = lastValue; + } + } + return costs[len2]; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/expression/filters/DigFilter.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/expression/filters/DigFilter.java new file mode 100644 index 0000000000..76b945da65 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/expression/filters/DigFilter.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.expression.filters; + +import java.util.List; +import java.util.Map; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.lib.filter.Filter; + +/** + * Custom Jinjava filter to dig into nested maps/lists. + * Supports negative indices for lists to access elements from the end. + * + * Usage: variable|dig("key1", "key2", 0) to access variable["key1"]["key2"][0] + * Also supports dot-notation in a single argument: variable|dig("key1.key2.0") + */ +@NonNullByDefault +public class DigFilter implements Filter { + @Override + public String getName() { + return "dig"; + } + + @Override + @NonNullByDefault({}) + public @Nullable Object filter(@Nullable Object var, JinjavaInterpreter interpreter, String... args) { + Object current = var; + + // Expand any dot-notation args (e.g. "a.b.c") into separate key segments + java.util.List keys = new java.util.ArrayList<>(); + if (args != null) { + for (String arg : args) { + if (arg != null && arg.contains(".")) { + String[] parts = arg.split("\\."); + for (String p : parts) { + keys.add(p); + } + } else { + keys.add(arg); + } + } + } + + for (Object key : keys) { // Changed to Object to be more flexible + if (current == null) { + return null; + } + + if (current instanceof Map) { + // Java Maps (like HashMap) can have a null key + current = ((Map) current).get(key); + } else if (current instanceof List) { + if (key == null) { + return null; // A list index cannot be null + } + current = getFromList((List) current, key); + } else { + return null; + } + } + return current; + } + + private @Nullable Object getFromList(List list, Object key) { + try { + int index; + if (key instanceof Integer intKey) { + index = intKey; + } else { + index = Integer.parseInt(key.toString()); + } + + int size = list.size(); + if (index < 0) { + index = size + index; + } + + if (index >= 0 && index < size) { + return list.get(index); + } + } catch (NumberFormatException ignored) { + } + return null; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/expression/filters/LabelFilter.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/expression/filters/LabelFilter.java new file mode 100644 index 0000000000..7210610711 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/expression/filters/LabelFilter.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.expression.filters; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.core.util.StringUtils; + +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import com.hubspot.jinjava.lib.filter.Filter; + +/** + * Custom Jinjava filter to convert a string to a label. + * + *

+ * It inserts spaces before capital letters, replaces '_' , ':' and '-' with spaces, + * and converts the result to title case. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class LabelFilter implements Filter { + @Override + public String getName() { + return "label"; + } + + @Override + @NonNullByDefault({}) + public @Nullable Object filter(@Nullable Object var, JinjavaInterpreter interpreter, String... args) { + if (var == null) { + return null; + } + String input = var.toString(); + String result = input // + .replaceAll("([a-z])([A-Z])", "$1 $2") // Insert space before capital letters + .replaceAll("([A-Z])([A-Z][a-z])", "$1 $2") // Handle cases like "HTTPServer" -> "HTTP Server" + .replaceAll("([A-Za-z])([0-9])", "$1 $2") // Insert space between letters and numbers + .replaceAll("([0-9])([A-Za-z])", "$1 $2") // Insert space between numbers and letters + .replaceAll("[_:-]+", " ") // Replace '_' , ':' and '-' with spaces + .replaceAll("\\s+", " "); // Replace multiple spaces with a single space + // Title case + result = StringUtils.capitalizeByWhitespace(result); + return result; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/IfPlaceholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/IfPlaceholder.java new file mode 100644 index 0000000000..52edc67931 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/IfPlaceholder.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.placeholders; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * The {@link IfPlaceholder} represents an object constructed from an !if node + * to be processed by the {@link org.openhab.io.yamlcomposer.internal.YamlComposer}. + * + * @param value The constructed object of the node containing the raw argument for the if placeholder + * @param sourceLocation Description of the source location for logging purposes + * + * @author Jimmy Tanagra - Initial contribution + */ +@SuppressWarnings("null") +@NonNullByDefault +public record IfPlaceholder(@Nullable Object value, + @NonNull String sourceLocation) implements InterpolablePlaceholder { + + @Override + public IfPlaceholder recreate(@Nullable Object newValue, String location) { + return new IfPlaceholder(newValue, location); + } + + @Override + public boolean eagerArgumentProcessing() { + // Do not process the arguments before resolving the conditions + // so that !include within unmet conditions are not processed + return false; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/IncludePlaceholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/IncludePlaceholder.java new file mode 100644 index 0000000000..0c61779633 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/IncludePlaceholder.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.placeholders; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * The {@link IncludePlaceholder} represents an object constructed from an !include node + * to be processed by the {@link org.openhab.io.yamlcomposer.internal.YamlComposer}. + * + * @param value The constructed object of the node containing the raw argument for the include placeholder + * @param sourceLocation Description of the source location for logging purposes + * + * @author Jimmy Tanagra - Initial contribution + */ +@SuppressWarnings("null") +@NonNullByDefault +public record IncludePlaceholder(@Nullable Object value, + @NonNull String sourceLocation) implements InterpolablePlaceholder { + + @Override + public IncludePlaceholder recreate(@Nullable Object newValue, String location) { + return new IncludePlaceholder(newValue, location); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/InsertPlaceholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/InsertPlaceholder.java new file mode 100644 index 0000000000..baa759f264 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/InsertPlaceholder.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.placeholders; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * The {@link InsertPlaceholder} represents an object constructed from an !insert node + * to be processed by the {@link org.openhab.io.yamlcomposer.internal.YamlComposer}. + * + * @param value The constructed object of the node containing the raw argument for the insert placeholder + * @param sourceLocation Description of the source location for logging purposes + * + * @author Jimmy Tanagra - Initial contribution + */ +@SuppressWarnings("null") +@NonNullByDefault +public record InsertPlaceholder(@Nullable Object value, + @NonNull String sourceLocation) implements InterpolablePlaceholder { + + @Override + public InsertPlaceholder recreate(@Nullable Object newValue, String location) { + return new InsertPlaceholder(newValue, location); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/InterpolablePlaceholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/InterpolablePlaceholder.java new file mode 100644 index 0000000000..26d3d4b723 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/InterpolablePlaceholder.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.placeholders; + +import java.util.Objects; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * Common abstraction for placeholders to allow generic access to their + * argument data while preserving typed, record-based implementations. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public interface InterpolablePlaceholder> extends Placeholder { + + @Nullable + Object value(); + + /** + * The factory method each record must implement to allow the generic withValue() + * method to create new instances of the correct type. + */ + R recreate(@Nullable Object newValue, String location); + + /** + * Convenience method to create a new instance of the placeholder with a new value but the same source location. + * + * @param newValue the new value for the placeholder + * @return a new instance of the placeholder with the given value and the same source location + */ + @SuppressWarnings("unchecked") + default R withValue(@Nullable Object newValue) { + if (Objects.equals(this.value(), newValue)) { + return (R) this; + } + // Direct method call - no reflection needed + return recreate(newValue, this.sourceLocation()); + } + + /** + * By default, we want to eagerly process the arguments of placeholders before processing the placeholder itself + * since most of them are just simple wrappers. + * + * However, some placeholders like !if need to resolve their conditions first before processing their arguments + * to avoid processing arguments that are not relevant (e.g. !include within unmet conditions). + * + * Implementations can override this method to change this behavior. + * + * @return true if the arguments should be processed before processing the placeholder, false otherwise + */ + default boolean eagerArgumentProcessing() { + return true; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/MergeKeyPlaceholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/MergeKeyPlaceholder.java new file mode 100644 index 0000000000..3522300d90 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/MergeKeyPlaceholder.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.placeholders; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * The {@link MergeKeyPlaceholder} replaces the merge key (<<) in a mapping node + * as a unique object, so that multiple merge keys can be processed later. + * + * @param value The value associated with the merge key placeholder ('<<' - not used) + * @param sourceLocation Description of the source location for logging purposes + * + * @author Jimmy Tanagra - Initial contribution + */ +@SuppressWarnings("null") +@NonNullByDefault +public record MergeKeyPlaceholder(@Nullable Object value, + @NonNull String sourceLocation) implements InterpolablePlaceholder { + + @Override + public MergeKeyPlaceholder recreate(@Nullable Object newValue, String location) { + return new MergeKeyPlaceholder(newValue, location); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/Placeholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/Placeholder.java new file mode 100644 index 0000000000..7bb60e742c --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/Placeholder.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.placeholders; + +import org.eclipse.jdt.annotation.NonNullByDefault; + +/** + * Common abstraction for placeholders. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public interface Placeholder { + /** + * Source location string for logging and diagnostics. + * + * @return non-null source location + */ + String sourceLocation(); +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/RemovePlaceholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/RemovePlaceholder.java new file mode 100644 index 0000000000..1746922b6f --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/RemovePlaceholder.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.placeholders; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * The {@link RemovePlaceholder} represents an object constructed from an !remove node + * to be processed by the {@link org.openhab.io.yamlcomposer.internal.YamlComposer}. + * + * @param value The value associated with the remove placeholder (not used) + * @param sourceLocation The source location of the remove placeholder for logging purposes + * + * @author Jimmy Tanagra - Initial contribution + */ +@SuppressWarnings("null") +@NonNullByDefault +public record RemovePlaceholder(@Nullable Object value, + @NonNull String sourceLocation) implements InterpolablePlaceholder { + + @Override + public RemovePlaceholder recreate(@Nullable Object newValue, String location) { + return new RemovePlaceholder(newValue, location); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/ReplacePlaceholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/ReplacePlaceholder.java new file mode 100644 index 0000000000..1271c165a2 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/ReplacePlaceholder.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.placeholders; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * The {@link ReplacePlaceholder} represents an object constructed from a !replace node + * to be processed by the {@link org.openhab.io.yamlcomposer.internal.YamlComposer}. + * + * @param value The value associated with the replace placeholder + * @param sourceLocation Description of the source location for logging purposes + * + * @author Jimmy Tanagra - Initial contribution + */ +@SuppressWarnings("null") +@NonNullByDefault +public record ReplacePlaceholder(@Nullable Object value, + @NonNull String sourceLocation) implements InterpolablePlaceholder { + + @Override + public ReplacePlaceholder recreate(@Nullable Object newValue, String location) { + return new ReplacePlaceholder(newValue, location); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/SubstitutionPlaceholder.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/SubstitutionPlaceholder.java new file mode 100644 index 0000000000..f7d3319e82 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/placeholders/SubstitutionPlaceholder.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.placeholders; + +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; + +/** + * The {@link SubstitutionPlaceholder} represents a deferred string interpolation constructed from a !sub + * node to be processed by the {@link org.openhab.io.yamlcomposer.internal.YamlComposer}. + * + *

+ * It preserves the raw scalar value and the optional variable name that defines delimiter pattern for + * interpolation. + * + * @param value The raw string value containing variable interpolation patterns + * @param patternName The variable name containing delimiter specification in the form .. + * @param sourceLocation Description of the source location for logging purposes + * + * @author Jimmy Tanagra - Initial contribution + */ +@SuppressWarnings("null") +@NonNullByDefault +public record SubstitutionPlaceholder(String value, @Nullable String patternName, + @NonNull String sourceLocation) implements Placeholder { +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/FragmentUtils.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/FragmentUtils.java new file mode 100644 index 0000000000..006ce2cdf4 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/FragmentUtils.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.processors; + +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.placeholders.InterpolablePlaceholder; + +/** + * The {@link FragmentUtils} provides common functionality + * for dealing with {@link org.openhab.io.yamlcomposer.internal.placeholders.IncludePlaceholder} and + * {@link org.openhab.io.yamlcomposer.internal.placeholders.InsertPlaceholder} parameters. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class FragmentUtils { + + public static record Parameters(@Nullable String name, Map varsMap) { + } + + /** + * Parses the parameters from an IncludePlaceholder or InsertPlaceholder. + */ + public static @Nullable Parameters parseParameters(InterpolablePlaceholder placeholder, String objectName) { + return switch (placeholder.value()) { + case null -> null; + case String name -> parseStringParameters(name, objectName); + case Map paramsMap -> parseMapParameters(paramsMap, objectName); + default -> null; + }; + } + + private static @Nullable Parameters parseMapParameters(Map paramsMap, String objectName) { + if (!(paramsMap.get(objectName) instanceof String name)) { + return new Parameters(null, Map.of()); + } + if (!(paramsMap.get("vars") instanceof Map varsMap)) { + return new Parameters(name, Map.of()); + } + + Map vars = varsMap.entrySet().stream().collect(LinkedHashMap::new, + (m, v) -> m.put(String.valueOf(v.getKey()), v.getValue()), LinkedHashMap::putAll); + return new Parameters(name, vars); + } + + private static Parameters parseStringParameters(String input, String objectName) { + if (input.isBlank()) { + return new Parameters(null, Map.of()); + } + + // 1. Separate Name and Query without substrings yet + int queryStart = input.indexOf('?'); + + // Process the Fragment Name + String rawName = (queryStart == -1) ? input : input.substring(0, queryStart); + String decodedName = safeDecode(rawName.trim()); + @Nullable + String finalName = decodedName.isEmpty() ? null : decodedName; + + // 2. Early exit if no query + if (queryStart == -1 || queryStart == input.length() - 1) { + return new Parameters(finalName, Map.of()); + } + + // 3. Parse Query String efficiently + Map vars = new LinkedHashMap<>(); + int len = input.length(); + int pos = queryStart + 1; + + while (pos < len) { + int nextAmp = input.indexOf('&', pos); + int end = (nextAmp == -1) ? len : nextAmp; + + if (end > pos) { // Ignore empty pairs like '&&' + parsePair(input, pos, end, vars); + } + pos = end + 1; + } + + return new Parameters(finalName, vars); + } + + private static void parsePair(String input, int start, int end, Map vars) { + int eq = input.indexOf('=', start); + + // If eq is outside our current segment, there is no value + if (eq == -1 || eq >= end) { + String key = safeDecode(input.substring(start, end).trim()); + if (!key.isEmpty()) { + vars.put(key, Boolean.TRUE); + } + } else { + String key = safeDecode(input.substring(start, eq).trim()); + String val = safeDecode(input.substring(eq + 1, end)); + if (!key.isEmpty()) { + vars.put(key, val); + } + } + } + + private static String safeDecode(String str) { + if (str.indexOf('%') == -1 && str.indexOf('+') == -1) { + return str; // Fast path: nothing to decode + } + try { + return URLDecoder.decode(str, StandardCharsets.UTF_8); + } catch (Exception e) { + return str; + } + } + + /** + * Converts the Parameters back to a value object for IncludePlaceholder or InsertPlaceholder. + */ + public static @Nullable Object toValue(@Nullable String name, Map varsMap, + String objectName) { + if (varsMap.isEmpty()) { + return name; + } + if (name == null) { + return Map.of("vars", varsMap); + } + return Map.of(objectName, name, "vars", varsMap); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/IfProcessor.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/IfProcessor.java new file mode 100644 index 0000000000..011d6bdf58 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/IfProcessor.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.processors; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.BufferedLogger; +import org.openhab.io.yamlcomposer.internal.StringInterpolator; +import org.openhab.io.yamlcomposer.internal.core.RecursiveTransformer; +import org.openhab.io.yamlcomposer.internal.expression.ExpressionEvaluator; +import org.openhab.io.yamlcomposer.internal.placeholders.IfPlaceholder; + +/** + * Processor for resolving {@link IfPlaceholder} in YAML models. + * Evaluates conditions and returns the appropriate branch value. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class IfProcessor implements PlaceholderProcessor { + private final BufferedLogger logger; + + public IfProcessor(BufferedLogger logger) { + this.logger = logger; + } + + @Override + public Class getPlaceholderType() { + return IfPlaceholder.class; + } + + private record Branch(@Nullable Object condition, @Nullable Object thenValue) { + } + + private record Logic(List branches, @Nullable Object elseValue) { + } + + @Override + public @Nullable Object process(IfPlaceholder ifPlaceholder, RecursiveTransformer recursiveTransformer) { + Logic logic = switch (ifPlaceholder.value()) { + case null -> null; + case Map map -> parseFromMapDefinition(map, ifPlaceholder.sourceLocation()); + case List list -> parseFromListDefinition(list, ifPlaceholder.sourceLocation()); + default -> null; + }; + + if (logic == null) { + return null; + } + + for (Branch branch : logic.branches()) { + Object evaluated = switch (branch.condition()) { + case null -> "false"; // Treat null condition as false + case String s -> StringInterpolator.evaluateExpression(s, recursiveTransformer.getVariables(), + logger.getLogSession(), ifPlaceholder.sourceLocation()); + default -> branch.condition(); + }; + + if (ExpressionEvaluator.isTruthy(evaluated)) { + return branch.thenValue(); + } + } + return logic.elseValue(); + } + + private @Nullable Logic parseFromMapDefinition(Map map, String sourceLocation) { + Branch mainBranch = createBranch(map, false, false, sourceLocation); + if (mainBranch == null) { + return null; + } + + Object elseValue = map.get("else"); + + return new Logic(List.of(mainBranch), elseValue); + } + + private @Nullable Logic parseFromListDefinition(List list, String sourceLocation) { + List branches = new ArrayList<>(); + @Nullable + Object elseValue = null; + + for (int i = 0; i < list.size(); i++) { + Object item = list.get(i); + if (!(item instanceof Map map)) { + logger.warn("{} !if list item is not a mapping and will be ignored: {}.", sourceLocation, item); + continue; + } + + if (map.containsKey("else")) { + elseValue = map.get("else"); + if (map.containsKey("if") || map.containsKey("elseif") || map.containsKey("then")) { + logger.warn("{} !if sequence item at index {} should only contain 'else'. Other keys ignored.", + sourceLocation, i); + } + if (i < list.size() - 1) { + logger.warn("{} !if list has unreachable branches after the 'else' block.", sourceLocation); + } + break; // Else stops evaluation of further branches + } + + Branch branch = createBranch(map, i > 0, true, sourceLocation); + if (branch != null) { + branches.add(branch); + } + } + + if (branches.isEmpty()) { + logger.warn("{} !if sequence has no valid branches.", sourceLocation); + } + + return new Logic(branches, elseValue); + } + + /** + * Creates a Branch from the given map. + * + * @param map the map representing the branch + * @param isElseIf whether this branch is an "elseif" branch + * @param isList whether this is for a list (non-root) branch + * @return the created Branch, or null if creation failed + */ + private @Nullable Branch createBranch(Map map, boolean isElseIf, boolean isList, String sourceLocation) { + String key = isElseIf ? "elseif" : "if"; + boolean hasConditionKey = map.containsKey(key); + boolean hasValueKey = map.containsKey("then"); + Object rawCondition = map.get(key); // Will be null if missing OR explicitly null + Object rawValue = map.get("then"); + + if (rawCondition == null) { + rawCondition = "false"; // Treat explicit null as false + } + + // 1. Guard: Check existence and nullability + // Condition: must exist + // Value: must exist (can be null) + if (!hasValueKey || !hasConditionKey) { + String prefix = isList ? "branch " : ""; + String suffix = isList ? "Ignoring branch." : "Returning null."; + + String reason; + if (!hasConditionKey && !hasValueKey) { + reason = "is empty (missing both '" + key + "' and 'then' fields)"; + } else if (!hasConditionKey) { + reason = "is missing '" + key + "' field"; + } else { + reason = "is missing 'then' field"; + } + + logger.warn("{} !if {}{}. {}", sourceLocation, prefix, reason, suffix); + return null; + } + + // 2. Success: Value exists (even if null), Condition exists and is non-null + return new Branch(rawCondition, rawValue); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/IncludeProcessor.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/IncludeProcessor.java new file mode 100644 index 0000000000..3009a469da --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/IncludeProcessor.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.processors; + +import java.io.IOException; +import java.nio.file.FileSystemException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.BufferedLogger; +import org.openhab.io.yamlcomposer.internal.ComposerConfig; +import org.openhab.io.yamlcomposer.internal.YamlComposer; +import org.openhab.io.yamlcomposer.internal.YamlComposer.CacheEntry; +import org.openhab.io.yamlcomposer.internal.core.RecursiveTransformer; +import org.openhab.io.yamlcomposer.internal.placeholders.IncludePlaceholder; +import org.snakeyaml.engine.v2.exceptions.MarkedYamlEngineException; +import org.snakeyaml.engine.v2.exceptions.YamlEngineException; + +/** + * Processor for resolving {@link IncludePlaceholder} in YAML models + * into the included file content. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class IncludeProcessor implements PlaceholderProcessor { + + private final BufferedLogger logger; + private final Path basePath; + private final Set includeStack; + private final Consumer includeCallback; + private final ConcurrentHashMap includeCache; + + /** + * Creates a new IncludeProcessor. + * + * @param basePath the base path to resolve relative includes against + * @param includeStack the stack of currently included files + * @param includeCallback the callback to invoke when a file is included + * @param includeCache the cache for storing included file contents + * @param logger the logger to use for logging messages + */ + public IncludeProcessor(Path basePath, Set includeStack, Consumer includeCallback, + ConcurrentHashMap includeCache, BufferedLogger logger) { + this.logger = logger; + this.basePath = basePath.toAbsolutePath().normalize(); + this.includeStack = includeStack; + this.includeCallback = includeCallback; + this.includeCache = includeCache; + } + + @Override + public Class getPlaceholderType() { + return IncludePlaceholder.class; + } + + /** + * Resolves an {@link IncludePlaceholder}, loads the referenced file + * (recursively following any nested includes), and returns the fully expanded + * content. + * + * @param placeholder the placeholder to process + * @param recursiveTransformer the recursive transformer to use + * @return the expanded content + */ + @Override + public @Nullable Object process(IncludePlaceholder placeholder, RecursiveTransformer recursiveTransformer) { + + FragmentUtils.Parameters params = FragmentUtils.parseParameters(placeholder, "file"); + if (params == null) { + logger.warn("{} Failed to process !include: invalid parameters", placeholder.sourceLocation()); + return null; + } + + Object fileNameObj = params.name(); + if (fileNameObj == null || String.valueOf(fileNameObj).isBlank()) { + logger.warn("{} Failed to process !include: missing 'file' parameter", placeholder.sourceLocation()); + return null; + } + + Path includePath = resolvePathPlaceholder(String.valueOf(fileNameObj), placeholder.sourceLocation()) + .toAbsolutePath().normalize(); + Path configRoot = ComposerConfig.configRoot(); + Path includePathRelative = includePath.startsWith(configRoot) ? configRoot.relativize(includePath) + : includePath; + + // Handle parameters and variables + Map includeVariables = new HashMap<>(recursiveTransformer.getVariables()); + includeVariables.putAll(params.varsMap()); // params override current variables + + try { + YamlComposer includeComposer = new YamlComposer(includePath, includeVariables, includeStack, + includeCallback, logger.getLogSession(), includeCache); + includeCallback.accept(includePath); + return includeComposer.load(); + } catch (YamlEngineException | IOException e) { + logIncludeError(placeholder, String.valueOf(fileNameObj), includePathRelative, e); + return null; + } + } + + /** + * Resolves the file path from the !include statement, handling any placeholders + * and providing detailed logging if resolution fails. + * + * Placeholders starting with '@' are resolved relative to OPENHAB_CONF, + * while those starting with '$' are resolved relative to OPENHAB_CONF/yamlcomposer. + * (i.e., {@link ComposerConfig#sourceRoot()}). + * + * This allows for flexible referencing of files + * within the configuration structure. + * + * If no placeholder prefix is used, absolute paths are kept as-is and relative paths are resolved against the + * including file's directory. + * + * @param includeFileName the file name from the !include statement, which may contain placeholders + * @param sourceLocation the source location for logging purposes + * @return the resolved Path to the included file, attempting to resolve placeholders if present + */ + private Path resolvePathPlaceholder(String includeFileName, String sourceLocation) { + char prefix = includeFileName.charAt(0); + Path root = switch (prefix) { + case '@' -> ComposerConfig.configRoot(); + case '$' -> ComposerConfig.sourceRoot(); + default -> null; + }; + + if (root != null) { + String cleanedPath = includeFileName.replaceFirst("^[@$]/*", ""); + return root.resolve(cleanedPath); + } + + Path includeFile = Path.of(includeFileName); + return includeFile.isAbsolute() ? includeFile.normalize() : basePath.resolve(includeFile); + } + + private void logIncludeError(IncludePlaceholder p, String name, Path path, Exception e) { + String location = (e instanceof MarkedYamlEngineException me) + ? me.getProblemMark().map(m -> "%d:%d".formatted(m.getLine() + 1, m.getColumn() + 1)).orElse(null) + : null; + + String pathWithLocation = (location == null) ? path.toString() : path + ":" + location; + String msg = (e instanceof IOException ioe) ? getFriendlyMessage(ioe) : e.getMessage(); + + logger.warn("{} Failed to process !include '{}'\n{} {}", p.sourceLocation(), name, pathWithLocation, msg); + } + + private static @Nullable String getFriendlyMessage(Exception e) { + if (e instanceof FileSystemException fse) { + // If the JDK provided a specific reason string, use it + if (fse.getReason() != null && !fse.getReason().isBlank()) { + return fse.getReason(); + } + + // Otherwise, use our "Sentence case" class name logic + String name = e.getClass().getSimpleName().replace("Exception", ""); + String spaced = name.replaceAll("([a-z])([A-Z])", "$1 $2").toLowerCase().trim(); + return (spaced.isBlank()) ? "File system error" + : Character.toUpperCase(spaced.charAt(0)) + spaced.substring(1); + } + return e.getMessage(); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/InsertProcessor.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/InsertProcessor.java new file mode 100644 index 0000000000..bc59710c8e --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/InsertProcessor.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.processors; + +import java.util.Map; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.BufferedLogger; +import org.openhab.io.yamlcomposer.internal.core.ProcessingPhase; +import org.openhab.io.yamlcomposer.internal.core.RecursiveTransformer; +import org.openhab.io.yamlcomposer.internal.placeholders.InsertPlaceholder; + +/** + * Processor for resolving {@link InsertPlaceholder} in YAML models + * into template content with local variable substitution. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class InsertProcessor implements PlaceholderProcessor { + + private final Map templates; + private final BufferedLogger logger; + + /** + * Creates a new InsertProcessor. + * + * @param templates the map of available templates for insertion + * @param logger the logger to use for logging messages + */ + public InsertProcessor(Map templates, BufferedLogger logger) { + this.templates = templates; + this.logger = logger; + } + + @Override + public Class getPlaceholderType() { + return InsertPlaceholder.class; + } + + /** + * Resolves an {@link InsertPlaceholder} recursively + * following any nested inserts, and returns the fully expanded content. + */ + @Override + public @Nullable Object process(InsertPlaceholder placeholder, RecursiveTransformer recursiveTransformer) { + FragmentUtils.Parameters params = FragmentUtils.parseParameters(placeholder, "template"); + if (params == null) { + logger.warn("{} Failed to process !insert: invalid parameters", placeholder.sourceLocation()); + return null; + } + + String templateName = params.name(); + if (templateName == null || templateName.isBlank()) { + logger.warn("{} Failed to process !insert: missing template name", placeholder.sourceLocation()); + return null; + } + + Object templateObj = templates.get(templateName); + if (templateObj == null) { + logger.warn("{} Failed to process !insert '{}': template not found", placeholder.sourceLocation(), + templateName); + return null; + } + + // The substitution placeholders in the template are resolved using the templateVariables context + // unlike any other processing which uses the main variables map + Map templateVariables = params.varsMap(); + RecursiveTransformer localTransformer = recursiveTransformer.withOverrideVariables(templateVariables); + // Keep package override placeholders (!remove/!replace) intact so they are only + // applied in the dedicated PACKAGE_OVERRIDES phase. + Object resolvedTemplate = localTransformer.transform(templateObj, ProcessingPhase.STANDARD); + return resolvedTemplate; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/PlaceholderProcessor.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/PlaceholderProcessor.java new file mode 100644 index 0000000000..8773065117 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/PlaceholderProcessor.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.processors; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.core.RecursiveTransformer; +import org.openhab.io.yamlcomposer.internal.placeholders.Placeholder; + +/** + * The {@link PlaceholderProcessor} processes placeholder instances in YAML models. + * + * @param The type of placeholder to be processed + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public interface PlaceholderProcessor { + + @Nullable + Object process(T placeholder, RecursiveTransformer recursiveTransformer); + + Class getPlaceholderType(); +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/RemoveProcessor.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/RemoveProcessor.java new file mode 100644 index 0000000000..d9a3524668 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/RemoveProcessor.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.processors; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.core.RecursiveTransformer; +import org.openhab.io.yamlcomposer.internal.core.RemovalSignal; +import org.openhab.io.yamlcomposer.internal.placeholders.RemovePlaceholder; + +/** + * The {@link RemoveProcessor} processes {@link RemovePlaceholder} instances in YAML models. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class RemoveProcessor implements PlaceholderProcessor { + + @Override + public Class getPlaceholderType() { + return RemovePlaceholder.class; + } + + @Override + public @Nullable Object process(RemovePlaceholder placeholder, RecursiveTransformer recursiveTransformer) { + return RemovalSignal.REMOVE; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/ReplaceProcessor.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/ReplaceProcessor.java new file mode 100644 index 0000000000..e05f7340a6 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/ReplaceProcessor.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.processors; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.core.RecursiveTransformer; +import org.openhab.io.yamlcomposer.internal.placeholders.ReplacePlaceholder; + +/** + * The {@link ReplaceProcessor} processes {@link ReplacePlaceholder} instances in YAML models. + * It simply returns the value contained within the placeholder. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class ReplaceProcessor implements PlaceholderProcessor { + + @Override + public Class getPlaceholderType() { + return ReplacePlaceholder.class; + } + + @Override + public @Nullable Object process(ReplacePlaceholder placeholder, RecursiveTransformer recursiveTransformer) { + return placeholder.value(); + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/SubstitutionProcessor.java b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/SubstitutionProcessor.java new file mode 100644 index 0000000000..5bdbc32dc3 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/java/org/openhab/io/yamlcomposer/internal/processors/SubstitutionProcessor.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal.processors; + +import java.util.Map; +import java.util.regex.Pattern; + +import org.eclipse.jdt.annotation.NonNullByDefault; +import org.eclipse.jdt.annotation.Nullable; +import org.openhab.io.yamlcomposer.internal.BufferedLogger; +import org.openhab.io.yamlcomposer.internal.StringInterpolator; +import org.openhab.io.yamlcomposer.internal.core.RecursiveTransformer; +import org.openhab.io.yamlcomposer.internal.placeholders.SubstitutionPlaceholder; + +/** + * Processor for handling variable substitutions by resolving {@code SubstitutionPlaceholder} in YAML models + * into interpolated strings. + * + * @author Jimmy Tanagra - Initial contribution + */ +@NonNullByDefault +public class SubstitutionProcessor implements PlaceholderProcessor { + + private final BufferedLogger logger; + + public SubstitutionProcessor(BufferedLogger logger) { + this.logger = logger; + } + + @Override + public Class getPlaceholderType() { + return SubstitutionPlaceholder.class; + } + + /** + * Recursively resolve all SubstitutionPlaceholders in a value (map, list, or scalar). + * using the RecursiveTransformer's variables as the substitution context. + * + * @param value The value to process + * @return The processed value with substitutions applied + */ + @Override + public @Nullable Object process(SubstitutionPlaceholder placeholder, RecursiveTransformer recursiveTransformer) { + return process(placeholder, recursiveTransformer.getVariables()); + } + + /** + * Recursively resolve all SubstitutionPlaceholders in a value (map, list, or scalar) + * using the provided context for substitutions. + * + * This overload is needed by {@link InsertProcessor} to apply substitutions + * with a custom variable context when processing templates. + * + * @param value The value to process + * @param context The variable context for substitutions + * @return The processed value with substitutions applied + */ + public @Nullable Object process(SubstitutionPlaceholder placeholder, Map context) { + Pattern pattern = resolvePattern(placeholder, context); + return StringInterpolator.interpolate(placeholder.value(), pattern, context, logger.getLogSession(), + placeholder.sourceLocation()); + } + + private Pattern resolvePattern(SubstitutionPlaceholder placeholder, Map context) { + String patternName = placeholder.patternName(); + if (patternName == null || patternName.isBlank()) { + return StringInterpolator.DEFAULT_SUBSTITUTION_PATTERN; + } + + Object rawPatternSpec = context.get(patternName); + if (!(rawPatternSpec instanceof String patternSpec) || patternSpec.isBlank()) { + logger.warn("{} Undefined or invalid pattern variable '{}' for !sub tag; using default pattern.", + placeholder.sourceLocation(), patternName); + return StringInterpolator.DEFAULT_SUBSTITUTION_PATTERN; + } + + Pattern compiled = StringInterpolator.compilePatternSpec(patternSpec); + if (compiled == null) { + logger.warn("{} Invalid pattern specification '{}' in variable '{}' for !sub tag; using default pattern.", + placeholder.sourceLocation(), patternSpec, patternName); + return StringInterpolator.DEFAULT_SUBSTITUTION_PATTERN; + } + + return compiled; + } +} diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/resources/OH-INF/addon/addon.xml b/bundles/org.openhab.io.yamlcomposer/src/main/resources/OH-INF/addon/addon.xml new file mode 100644 index 0000000000..1eed632225 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/resources/OH-INF/addon/addon.xml @@ -0,0 +1,13 @@ + + + + misc + YAML Composer + This add-on processes YAML with extended syntax (includes, packages, variables) and outputs fully resolved + plain YAML for openHAB YAML Configuration. + local + + org.openhab.io.yamlcomposer + diff --git a/bundles/org.openhab.io.yamlcomposer/src/main/resources/OH-INF/i18n/yamlcomposer.properties b/bundles/org.openhab.io.yamlcomposer/src/main/resources/OH-INF/i18n/yamlcomposer.properties new file mode 100644 index 0000000000..e46a531bd8 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/main/resources/OH-INF/i18n/yamlcomposer.properties @@ -0,0 +1,4 @@ +# add-on + +addon.yamlcomposer.name = YAML Composer +addon.yamlcomposer.description = This add-on processes YAML with extended syntax (includes, packages, variables) and outputs fully resolved plain YAML for openHAB YAML Configuration. diff --git a/bundles/org.openhab.io.yamlcomposer/src/test/java/org/openhab/io/yamlcomposer/internal/YamlComposerTest.java b/bundles/org.openhab.io.yamlcomposer/src/test/java/org/openhab/io/yamlcomposer/internal/YamlComposerTest.java new file mode 100644 index 0000000000..cc7e0a10c5 --- /dev/null +++ b/bundles/org.openhab.io.yamlcomposer/src/test/java/org/openhab/io/yamlcomposer/internal/YamlComposerTest.java @@ -0,0 +1,3263 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.emptyOrNullString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.everyItem; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.IOException; +import java.lang.reflect.Array; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.Collections; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import org.eclipse.jdt.annotation.Nullable; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.openhab.io.yamlcomposer.internal.YamlComposer.CacheEntry; +import org.openhab.io.yamlcomposer.internal.placeholders.Placeholder; + +/** + * The {@link YamlComposerTest} contains tests for the {@link YamlComposer} class. + * + * @author Jimmy Tanagra - Initial contribution + */ +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +public class YamlComposerTest { + private static final Path SOURCE_PATH = Path.of("src/test/resources/model/composer"); + private @Nullable TestInfo currentTest = null; // so loadFixture can tell which test is calling it + + private @TempDir @Nullable Path sharedTempDir = null; // Initialized to avoid null warning + + // logSession is used by loadFixture to capture logs during YAML loading, + // which can be helpful for debugging test failures + private final LogSession logSession = new LogSession(); + + @BeforeEach + void setup(TestInfo testInfo) { + this.currentTest = testInfo; + } + + @AfterEach + void tearDown() { + logSession.close(); // This will flush logs to the console automatically + } + + @Nested + class TestInfrastructure { + @Test + @DisplayName("Extracts values from deeply nested maps") + void extractsValuesFromDeeplyNestedMaps() { + Map data = Map.of("top", Map.of("level1", Map.of("level2", "value"))); + + assertThat(getNestedValue(data, "top", "level1", "level2"), equalTo("value")); + } + + @Test + @DisplayName("Returns null when nested keys are missing") + void returnsNullWhenNestedKeysAreMissing() { + Map data = Map.of("top", Map.of("level1", "value")); + + assertNull(getNestedValue(data, "top", "nonexistent")); + assertNull(getNestedValue(data, "missing", "level1")); + } + } + + @Nested + class YamlParsing { + @Test + @DisplayName("Allows null values") + void allowsNullValues() throws IOException { + assertThat(loadYamlValue(""), is(nullValue())); + assertThat(loadYamlValue("# Comment"), is(nullValue())); + assertThat(loadYamlValue("null"), is(nullValue())); + assertThat(loadYamlValue("a: null"), equalTo(Collections.singletonMap("a", null))); + assertThat(loadYamlValue("null: null"), equalTo(Collections.emptyMap())); + assertThat(loadYamlValue("- null"), equalTo(Collections.emptyList())); + } + + @Test + @DisplayName("Parses true and false as boolean") + void parsesTrueAndFalseAsBoolean() throws IOException { + assertThat(loadYamlValue("true"), equalTo(true)); + assertThat(loadYamlValue("True"), equalTo(true)); + assertThat(loadYamlValue("TRUE"), equalTo(true)); + + assertThat(loadYamlValue("false"), equalTo(false)); + assertThat(loadYamlValue("False"), equalTo(false)); + assertThat(loadYamlValue("FALSE"), equalTo(false)); + } + + @Test + @DisplayName("Treats boolean-like strings as regular strings") + void treatsBooleanLikeStringsAsRegularStrings() throws IOException { + for (String value : List.of("on", "On", "ON", "oN")) { + assertThat(loadYamlValue(value), equalTo(value)); + } + + for (String value : List.of("off", "Off", "OFF", "oFf")) { + assertThat(loadYamlValue(value), equalTo(value)); + } + + for (String value : List.of("yes", "Yes", "YES", "yEs")) { + assertThat(loadYamlValue(value), equalTo(value)); + } + + for (String value : List.of("no", "No", "NO", "nO")) { + assertThat(loadYamlValue(value), equalTo(value)); + } + } + + @Test + @DisplayName("Supports anchors and aliases") + void supportsAnchorsAndAliases() throws IOException { + @SuppressWarnings("unchecked") + Map data = (Map) loadYamlValue(""" + foo: &name bar + baz: *name + ? *name + : qux + """); + assertThat(data.get("baz"), equalTo("bar")); + assertThat(data.get("bar"), equalTo("qux")); + } + + @Test + @DisplayName("Handles self-referencing container") + @SuppressWarnings("null") + void handlesSelfReferencingContainer() throws IOException { + String yaml = """ + baz: &id001 + me: *id001 + """; + + Map data = loadYaml(yaml); + + assertThat(data.get("baz"), instanceOf(Map.class)); + @SuppressWarnings("unchecked") + Map baz = (Map) data.get("baz"); + + // The 'me' entry should reference the same Map instance (self-reference) + assertSame(baz, baz.get("me")); + } + } + + @Nested + class ModelCleanup { + @Test + @DisplayName("Removes preprocessing metadata") + void removesPreprocessingMetadata() throws IOException { + String yaml = """ + composer: + generate_resolved_file: false + + variables: + foo: bar + + templates: + sample_template: foo + + packages: + foo: {} + """; + + Map data = loadYaml(yaml); + assertThat(data, not(hasKey("composers"))); + assertThat(data, not(hasKey("variables"))); + assertThat(data, not(hasKey("templates"))); + assertThat(data, not(hasKey("packages"))); + } + + @Test + @DisplayName("Removes hidden keys") + @SuppressWarnings("null") + void removesHiddenKeys() throws IOException { + String yaml = ".energy_type: foo"; + Map data = loadYaml(yaml); + List keys = data.keySet().stream().map(Object::toString).collect(Collectors.toList()); + assertThat(keys, everyItem(not(startsWith(".")))); + } + + @Test + @DisplayName("Retains other keys") + void retainsOtherKeys() throws IOException { + String yaml = """ + version: 1 + items: a + things: b + other: c + """; + Map data = loadYaml(yaml); + assertThat(data.get("version"), equalTo(1)); + assertThat(data.get("items"), equalTo("a")); + assertThat(data.get("things"), equalTo("b")); + assertThat(data.get("other"), equalTo("c")); + } + + @Test + @DisplayName("Removes null keys in maps") + void removesNullKeysInMaps() throws IOException { + String yaml = "map: { null: value, key1: val1 }"; + Map data = loadYaml(yaml); + assertThat(data.get("map"), equalTo(Map.of("key1", "val1"))); + } + } + + @Nested + @DisplayName("Variables and Substitutions") + class VariablesAndSubstitutions { + + @Nested + @DisplayName("Resolution and Scoping") + class ResolutionAndScoping { + + @Test + @DisplayName("Resolves plain variables in simple string expressions") + void resolvesPlainVariables() throws IOException { + String yaml = """ + variables: + greeting: "Hello" + target: "World" + test: "${greeting}, ${target}!" + """; + + Map data = loadYaml(yaml); + + assertThat("Basic variable resolution should work in a simple string expression", + getNestedValue(data, "test"), is("Hello, World!")); + } + + @Test + @DisplayName("Supports defining variables at end of file") + void supportsVariablesDefinedAtEndOfFile() throws IOException { + String yaml = """ + test: + result: "${late_var}" + + variables: + late_var: "hoisted" + """; + + Map data = loadYaml(yaml); + + assertThat("Variables should be resolvable regardless of their position in the file", + getNestedValue(data, "test", "result"), is("hoisted")); + } + + @Test + @DisplayName("Resolves predefined system variables (__FILE__, __DIRECTORY__, etc)") + void resolvesPredefinedSystemVariables() throws IOException { + Path file = writeFixture("predefinedVars.inc.yaml", """ + file: "${__FILE__}" + filename: "${__FILE_NAME__}" + ext: "${__FILE_EXT__}" + path: "${__DIRECTORY__}" + openhab_conf: "${OPENHAB_CONF}" + """); + + Map data = loadFixture(file); + + assertThat("The filename variable should be extracted from the file name", data.get("filename"), + is("predefinedVars.inc")); + + assertThat("The extension variable should be extracted from the file extension", data.get("ext"), + is("yaml")); + + assertThat("The directory variable should point to the file's parent folder", (String) data.get("path"), + containsString(file.getParent().getFileName().toString())); + + assertThat("System environment variables must be injected into the resolution context", + (String) data.get("openhab_conf"), is(not(emptyOrNullString()))); + } + + @Test + @DisplayName("Protects predefined variables from user/include overrides") + void protectsPredefinedVariablesFromOverrides() throws IOException { + writeFixture("predefinedVarsOverride.inc.yaml", """ + filename: "${__FILE_NAME__}" + """); + + Path mainFile = writeFixture("predefinedVarsOverride.yaml", """ + variables: + __FILE_NAME__: "main_override" + + filename: "${__FILE_NAME__}" + + include: !include + file: predefinedVarsOverride.inc.yaml + vars: + __FILE_NAME__: "include_override" + """); + + Map data = loadFixture(mainFile); + + assertThat("System variables in the main file should ignore the local 'variables' block overrides", + data.get("filename"), is("predefinedVarsOverride")); + + assertThat("System variables in an include should ignore 'vars' passed via !include", + getNestedValue(data, "include", "filename"), is("predefinedVarsOverride.inc")); + } + + @Test + @DisplayName("Loads entire variables block from an !include file") + void loadsEntireVariablesBlockFromInclude() throws IOException { + writeFixture("vars_file.inc.yaml", """ + external_var: "from_file" + another_var: "hello" + """); + + Path mainFile = writeFixture("main.yaml", """ + variables: !include vars_file.inc.yaml + + result: "${external_var} ${another_var}" + """); + + Map data = loadFixture(mainFile); + + assertThat("Variables from the included file should be available in the global context", + getNestedValue(data, "result"), is("from_file hello")); + } + + @Test + @DisplayName("Loads value from an include file") + void loadsVariablesFromIncludeFiles() throws IOException { + writeFixture("variableFromInclude.inc.yaml", "qux"); + + Path mainFile = writeFixture("variableFromInclude.yaml", """ + variables: + foo: !include variableFromInclude.inc.yaml + + included_value: ${foo} + """); + + Map data = loadFixture(mainFile); + + assertThat("Simple inclusion should work", getNestedValue(data, "included_value"), is("qux")); + } + + @Test + @DisplayName("Loads value from an include files with vars") + void loadsVariablesFromIncludeFilesWithVars() throws IOException { + // This also tests that a simple scalar inside an include file + // will resolve substitutions using the inherited context. + writeFixture("include.inc.yaml", "${var}"); + + Path mainFile = writeFixture("main.yaml", """ + variables: + foo: !include + file: include.inc.yaml + vars: + var: qux + + included_value: ${foo} + """); + + Map data = loadFixture(mainFile); + + assertThat("Simple inclusion should work", getNestedValue(data, "included_value"), is("qux")); + } + + @Test + @DisplayName("Merge keys at variables TOP-LEVEL with !include placeholder") + void mergeKeysAtVariablesMapLevel() throws IOException { + writeFixture("base_variables.yaml", """ + base_template: from_base + """); + + Path mainFile = writeFixture("main.yaml", """ + variables: + <<: !include base_variables.yaml + + merged_variable: ${base_template} + """); + + Map data = loadFixture(mainFile); + + assertThat(getNestedValue(data, "merged_variable"), equalTo("from_base")); + } + + @Test + @DisplayName("Supports substitution within variables block (Recursive resolution)") + void supportsSubstitutionWithinVariablesBlock() throws IOException { + String yaml = """ + variables: + a: "root" + b: "${a}-to-middle" + c: "${b}-to-leaf" + + test: + result: "${c}" + """; + + Map data = loadYaml(yaml); + + assertThat("Variables must support self-referential resolution", getNestedValue(data, "test", "result"), + is("root-to-middle-to-leaf")); + } + } + + @Nested + @DisplayName("Substitution Syntax") + class SubstitutionSyntax { + + @Test + @DisplayName("Handles quoting and escaping in expressions") + void handlesQuotingAndEscaping() throws IOException { + String yaml = """ + variables: + foo: value1 + + plain: ${foo} + double_quoted: "${foo}" + single_quoted: '${foo}' + braces_in_double_quotes: "${'${}'}" + braces_in_single_quotes: '${"${}"}' + """; + Map data = loadYaml(yaml); + + assertThat(data.get("plain"), is("value1")); + assertThat(data.get("double_quoted"), is("value1")); + assertThat(data.get("single_quoted"), is("value1")); + assertThat(data.get("braces_in_double_quotes"), is("${}")); + assertThat(data.get("braces_in_single_quotes"), is("${}")); + } + + @Test + @DisplayName("Handles empty and null variable values") + void handlesEmptyAndNullValues() throws IOException { + String yaml = """ + variables: + nonnull: value1 + + empty: ${} + null_value: ${null} + null_string: "${null}" + padded: ${ nonnull } + """; + Map data = loadYaml(yaml); + + assertThat(data.get("empty"), is(nullValue())); + assertThat(data.get("null_value"), is(nullValue())); + assertThat(data.get("null_string"), is(nullValue())); + assertThat(data.get("padded"), is("value1")); + } + + @Test + @DisplayName("Handles empty maps and object comparisons") + void handlesEmptyMap() throws IOException { + String yaml = """ + empty_map: ${ {} } + compare_empty_maps: ${ {} == {} } + """; + Map data = loadYaml(yaml); + + assertThat(data.get("empty_map"), instanceOf(Map.class)); + assertThat((Map) data.get("empty_map"), is(anEmptyMap())); + assertThat(data.get("compare_empty_maps"), equalTo(true)); + } + + @Test + @DisplayName("Handles special characters in variable names via VARS lookup") + void handlesSpecialVariableNames() throws IOException { + String yaml = """ + variables: + "varname-with-dash": dashvalue + "varname with space": spacevalue + + vars_with_dash: ${VARS["varname-with-dash"]} + vars_with_space: ${VARS['varname with space']} + """; + Map data = loadYaml(yaml); + + assertThat(data.get("vars_with_dash"), is("dashvalue")); + assertThat(data.get("vars_with_space"), is("spacevalue")); + } + + @Test + @DisplayName("Handles basic data types and mathematical expressions") + void handlesDataTypesAndMath() throws IOException { + String yaml = """ + variables: + one: 1 + + int_const: ${1 + 1} + int_var_math: ${one * 5} + int_quoted: "${100}" + string: ${'100'} + string_quoted: "${'100'}" + """; + Map data = loadYaml(yaml); + + assertThat(data.get("int_const"), is(2)); + assertThat(data.get("int_var_math"), is(5)); + assertThat(data.get("int_quoted"), is(100)); + assertThat(data.get("string"), is("100")); + assertThat(data.get("string_quoted"), is("100")); + } + + @Test + @DisplayName("Navigates complex mapping and list structures") + void navigatesComplexStructures() throws IOException { + String yaml = """ + variables: + mapping: { foo: bar } + list: [item0] + map_lookup: ${mapping.foo} + list_lookup: ${list[0]} + """; + Map data = loadYaml(yaml); + + assertThat(data.get("map_lookup"), is("bar")); + assertThat(data.get("list_lookup"), is("item0")); + } + + @Test + @DisplayName("Allows execution of Java String methods within expressions") + void executesJavaObjectMethod() throws IOException { + String yaml = """ + repeated: ${'bar'.repeat(2) + 'ian'} + """; + Map data = loadYaml(yaml); + + assertThat(data.get("repeated"), is("barbarian")); + } + } + + @Nested + @DisplayName("Expression Filters") + class ExpressionFilters { + + @Nested + @DisplayName("Standard Filters") + class StandardFilters { + + @Test + @DisplayName("Applies default values to missing or empty variables") + void appliesDefaultValues() throws IOException { + String yaml = """ + variables: + exist: value1 + empty_value: "" + + exists: ${exist|default('fallback')} + missing: ${unknown|default('fallback')} + empty_loose: ${empty_value|default('fallback')} + empty_strict: ${empty_value|default('fallback', true)} + """; + Map data = loadYaml(yaml); + + assertThat(data.get("exists"), is("value1")); + assertThat(data.get("missing"), is("fallback")); + assertThat(data.get("empty_loose"), is("")); + assertThat(data.get("empty_strict"), is("fallback")); + } + } + + @Nested + @DisplayName("Custom Filters") + class CustomFilters { + + @Nested + @DisplayName("Label") + class Label { + + @ParameterizedTest(name = "[{index}] Label conversion: \"{0}\" -> \"{1}\"") + @CsvSource(delimiter = '|', value = { // + "foo bar | Foo Bar", // + "fooBar | Foo Bar", // + "foo_bar | Foo Bar", // + "foo---bar_:-baz | Foo Bar Baz", // + "LivingRoom_Light1Dimmer | Living Room Light 1 Dimmer", // + "HTTPServer | HTTP Server", // + "Multiple Spaces | Multiple Spaces", // + "StatusLED | Status LED" // + }) + void convertsStringsToHumanReadableLabels(String input, String expected) throws IOException { + String yaml = """ + expression: ${'%s' | label} + """.formatted(input); + Map data = loadYaml(yaml); + assertThat(data.get("expression"), is(expected)); + } + } + + @Nested + @DisplayName("Dig") + class Dig { + + @Test + @DisplayName("Resolves deep paths from a variable") + void resolvesDeepMapPath() throws IOException { + String yaml = """ + variables: + system: + network: + ip: "192.168.1.1" + + # Accessing 2 levels deep from the 'system' root + result: ${system | dig('network', 'ip')} + """; + + Map data = loadYaml(yaml); + assertThat(data.get("result"), is("192.168.1.1")); + } + + @Test + @DisplayName("Returns null when the middle of the path is missing") + void returnsNullForMissingMiddlePath() throws IOException { + String yaml = """ + variables: + system: + network: { dns: "8.8.8.8" } + + # 'proxy' does not exist inside 'network' + missing: ${system | dig('network', 'proxy', 'host')} + """; + + Map data = loadYaml(yaml); + assertThat(data.get("missing"), is(nullValue())); + assertThat(logSession.getTrackedWarnings(), empty()); + } + + @Test + @DisplayName("Accesses list elements inside a nested map") + void navigatesNestedLists() throws IOException { + String yaml = """ + variables: + hardware: + usb_ports: ["port_a", "port_b"] + + # Combining Map lookup and List index + result: ${hardware | dig('usb_ports', 1)} + """; + + Map data = loadYaml(yaml); + assertThat(data.get("result"), is("port_b")); + } + + @Test + @DisplayName("Navigates deep list-of-lists") + void navigatesDeepLists() throws IOException { + String yaml = """ + variables: + matrix: + coords: [[1, 2], [3, 4]] + + # Digging through two levels of arrays + val: ${matrix | dig('coords', 1, 0)} + """; + + Map data = loadYaml(yaml); + assertThat(data.get("val"), is(3)); + } + + @Test + @DisplayName("Supports dot-notation single-argument paths") + void supportsDotNotationSingleArg() throws IOException { + String yaml = """ + variables: + system: + network: + ip: "192.168.1.1" + + matrix: + coords: [[1, 2], [3, 4]] + + # Using dot-notation in a single dig argument + result_ip: ${system | dig('network.ip')} + val: ${matrix | dig('coords.1.0')} + """; + + Map data = loadYaml(yaml); + assertThat(data.get("result_ip"), is("192.168.1.1")); + assertThat(data.get("val"), is(3)); + } + + @Test + @DisplayName("Supports mixed dot-notation and separate args") + void supportsMixedDotAndSeparateArgs() throws IOException { + String yaml = """ + variables: + system: + network: + ip: "192.168.1.1" + + matrix: + coords: [[1, 2], [3, 4]] + + root: + nested: + arr: [10, 20] + + # Mixed: combine separate arg and dot-notation arg + res1: ${root | dig('nested', 'arr.1')} + # Mixed: dot-notation plus numeric arg + res2: ${matrix | dig('coords.1', 0)} + """; + + Map data = loadYaml(yaml); + assertThat(data.get("res1"), is(20)); + assertThat(data.get("res2"), is(3)); + } + } + } + } + + @Nested + @DisplayName("Error Handling") + class ErrorHandling { + @ParameterizedTest + @ValueSource(strings = { "${'}", "${\"}", "${'\"}", "${\"'}", "${${}}", "${${}" }) + @DisplayName("Correctly identifies substitution boundaries during syntax errors") + void correctlyIdentifiesSubstitutionBoundariesInSyntaxErrors(String expression) throws IOException { + String yaml = "expression: " + expression; + Map data = loadYaml(yaml); + Object expressionValue = data.get("expression"); + assertThat("The parser should pass the entire content inside the braces to the engine", expressionValue, + is(nullValue())); + + assertThat("Malformed expressions should still be handed to the engine and log a parse error", + logSession.getTrackedWarnings(), hasItem(containsString("Error parsing"))); + } + + @ParameterizedTest + @ValueSource(strings = { "${undefined_variable}", "${2 + foo}" }) + @DisplayName("Warns when expressions contain unresolved variables or tokens") + void warnsOnUnresolvedVariables(String expression) throws IOException { + String yaml = "test: " + expression; + loadYaml(yaml); + + assertThat("The engine should log a warning for undefined variables or tokens", + logSession.getTrackedWarnings(), hasItem(anyOf(containsString("Undefined variable")))); + } + + @Test + @DisplayName("Provides spelling suggestions for misspelled variables") + void providesSuggestionsForMisspelledVariables() throws IOException { + String yaml = """ + variables: + correct_name: value + + test: ${corret_name} + """; + + loadYaml(yaml); + + assertThat("The engine should suggest similarly named variables for misspellings", + logSession.getTrackedWarnings(), hasItem(containsString("Did you mean 'correct_name'?"))); + } + } + + @Nested + @DisplayName("Null and Undefined Handling") + class NullHandling { + @Test + @DisplayName("Removes null elements from lists") + void removesNullListElements() throws IOException { + String yaml = "list: ${[ undefined_variable, null, 'normal string' ]}"; + Map data = loadYaml(yaml); + assertThat(data.get("list"), equalTo(List.of("normal string"))); + } + } + + @Nested + @DisplayName("Custom Delimiters") + class CustomDelimiters { + @Test + @DisplayName("Supports varying delimiter styles (brackets, parenthesis, at-symbols)") + void supportsVaryingDelimiterStyles() throws IOException { + String yaml = """ + variables: + foo: bar + bracket_pattern: "$[[..]]" + parenthesis_pattern: "$((..))" + at_symbol_pattern: "@[..]" + bracket: !sub:bracket_pattern "$[[foo]]" + parenthesis: !sub:parenthesis_pattern "$((foo))" + at_symbol: !sub:at_symbol_pattern "@[foo]" + """; + + Map data = loadYaml(yaml); + assertThat(data.get("bracket"), equalTo("bar")); + assertThat(data.get("parenthesis"), equalTo("bar")); + assertThat(data.get("at_symbol"), equalTo("bar")); + } + + @Test + @DisplayName("Supports multiple occurrences of custom delimiters in one string") + void supportsMultipleOccurrences() throws IOException { + String yaml = """ + variables: + foo: bar + bracket_pattern: "$[[..]]" + multiple: !sub:bracket_pattern "$[[foo]]_$[[foo]]" + """; + + Map data = loadYaml(yaml); + assertThat(data.get("multiple"), equalTo("bar_bar")); + } + + @Test + @DisplayName("Handles empty custom patterns gracefully") + void supportsEmptyPattern() throws IOException { + String yaml = """ + variables: + bracket_pattern: "$[[..]]" + empty: !sub:bracket_pattern 'A$[[]]B' + """; + Map data = loadYaml(yaml); + assertThat(data.get("empty"), equalTo("AB")); + } + + @Test + @DisplayName("Maintains delimiter scope through nested YAML structures") + void maintainsDelimiterScopeThroughNestedStructures() throws IOException { + String yaml = """ + variables: + foo: bar + bracket_pattern: "$[[..]]" + at_symbol_pattern: "@[..]" + level1: !sub:bracket_pattern + data: "$[[foo]]" + level2: + data: "$[[foo]]" + level2_override: !sub:at_symbol_pattern + data: "@[foo]" + data2: "$[[foo]]" + """; + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "level1", "data"), equalTo("bar")); + assertThat(getNestedValue(data, "level1", "level2", "data"), equalTo("bar")); + assertThat(getNestedValue(data, "level1", "level2_override", "data"), equalTo("bar")); + assertThat(getNestedValue(data, "level1", "data2"), equalTo("bar")); + } + } + + @Nested + @DisplayName("Substitution Control") + class SubstitutionControl { + @Test + @DisplayName("!literal tag prevents interpolation of variable patterns") + void literalTagPreventsInterpolation() throws IOException { + String yaml = """ + variables: { foo: bar } + top: + enabled: ${foo} + disabled_branch: !literal + level2: ${foo} + """; + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "top", "enabled"), is("bar")); + assertThat(getNestedValue(data, "top", "disabled_branch", "level2"), is("${foo}")); + } + + @Test + @DisplayName("!sub tag re-enables substitutions inside a parent !literal boundary") + void subTagCanOverrideParentLiteralTag() throws IOException { + String yaml = """ + variables: { foo: bar } + top: !literal + disabled: ${foo} + re_enabled: !sub ${foo} + """; + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "top", "disabled"), is("${foo}")); + assertThat(getNestedValue(data, "top", "re_enabled"), is("bar")); + } + } + + @Nested + @DisplayName("Substitutions inside other tags") + class SubstitutionsInsideOtherTags { + + @Test + @DisplayName("Applies substitutions to scalar form !include paths") + void appliesSubstitutionsToScalarFormInclude() throws IOException { + writeFixture("target.yaml", "key: value"); + + String yaml = """ + variables: { my_path: 'target.yaml' } + test: + result: !include "${my_path}" + """; + + Map data = loadYaml(yaml); + assertThat(getNestedValue(data, "test", "result", "key"), is("value")); + } + + @Test + @DisplayName("Applies substitutions to block form !include (file and vars)") + void appliesSubstitutionsToBlockFormInclude() throws IOException { + writeFixture("target.inc.yaml", "result: 'processed-${inner}'"); + + String yaml = """ + variables: + path_var: 'target.inc.yaml' + val_var: 'from-parent' + + test: + result: + !include + file: "${path_var}" + vars: + inner: "${val_var}" + """; + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "test", "result", "result"), is("processed-from-parent")); + } + + @Test + @DisplayName("Applies substitutions to scalar form !insert template names") + void appliesSubstitutionsToScalarFormInsert() throws IOException { + String yaml = """ + variables: { tpl_name: 'my_template' } + templates: + my_template: "Success" + + test: + result: !insert "${tpl_name}" + """; + + Map data = loadYaml(yaml); + assertThat(getNestedValue(data, "test", "result"), is("Success")); + } + + @Test + @DisplayName("Applies substitutions to block form !insert (template and vars)") + void appliesSubstitutionsToBlockFormInsert() throws IOException { + String yaml = """ + variables: + tpl_var: 'my_template' + suffix_var: 'engine' + + templates: + my_template: "power-${suffix}" + + test: + result: + !insert + template: "${tpl_var}" + vars: + suffix: "${suffix_var}" + """; + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "test", "result"), is("power-engine")); + } + + @Test + @DisplayName("Substitutions penetrate through !replace tag boundaries") + void appliesSubstitutionsWithinReplaceBlocks() throws IOException { + String yaml = """ + variables: + moo: cow + packages: + things: + MyThing: + foo: { bar: baz } + things: + MyThing: + foo: !replace + qux: "${moo}" + """; + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "things", "MyThing", "foo", "qux"), equalTo("cow")); + } + } + } + + @Nested + @DisplayName("!if Tag") + class IfTag { + + @Nested + @DisplayName("Syntax Forms") + class SyntaxForms { + + @Test + @DisplayName("Resolves simple mapping form") + void resolvesMappingForm() throws IOException { + String yaml = """ + test: !if + if: true + then: "matched" + """; + + Map data = loadYaml(yaml); + + assertThat("Mapping form should resolve correctly when true", getNestedValue(data, "test"), + is("matched")); + } + + @Test + @DisplayName("Resolves sequence form with multiple branches") + void resolvesSequenceForm() throws IOException { + String yaml = """ + test: !if + - if: false + then: "first" + - elseif: true + then: "second" + - else: "third" + """; + + Map data = loadYaml(yaml); + + assertThat("Sequence form should pick the first truthy branch", getNestedValue(data, "test"), + is("second")); + } + + @Test + @DisplayName("Resolves to else value when no branches match") + void resolvesElseFallback() throws IOException { + String yaml = """ + test: !if + - if: false + then: "no" + - else: "fallback" + """; + + Map data = loadYaml(yaml); + + assertThat("Should return else value if all conditions fail", getNestedValue(data, "test"), + is("fallback")); + } + } + + @Nested + @DisplayName("Expression Evaluation") + class ExpressionEvaluation { + + @Test + @DisplayName("Evaluates direct expressions") + void evaluatesExpressionsInConditions() throws IOException { + String yaml = """ + variables: + num: 5 + test: !if + if: num > 3 + then: "greater" + else: "lesser" + """; + + Map data = loadYaml(yaml); + + assertThat("Should evaluate expression and resolve to 'greater'", getNestedValue(data, "test"), + is("greater")); + } + + @Test + @DisplayName("Evaluates expressions in ${} pattern") + void evaluatesSubExpressionsInConditions() throws IOException { + String yaml = """ + variables: + threshold: 10 + test: !if + if: ${threshold} > 5 + then: "high" + else: "low" + """; + Map data = loadYaml(yaml); + assertThat("Should evaluate expression in condition", getNestedValue(data, "test"), is("high")); + } + } + + @Nested + @DisplayName("Truthiness and Logic") + class Truthiness { + + @ParameterizedTest + @ValueSource(strings = { "[]", "{}", "0", "' '", "false", "'false'", "null" }) + @DisplayName("Treats empty collections and zero as falsy") + void handlesFalsyValues(String ifValue) throws IOException { + String yaml = "value: !if { if: %s, then: 'yes', else: 'no' }".formatted(ifValue); + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "value"), is("no")); + } + + @ParameterizedTest + @ValueSource(strings = { "[1]", "42", "true", "'true'", "\"'hello'\"", "{ key: value }", "[item]" }) + @DisplayName("Treats non-empty collections and strings as truthy") + void handlesTruthyValues(String ifValue) throws IOException { + String yaml = """ + value: !if + if: %s + then: 'yes' + else: 'no' + """.formatted(ifValue); + ; + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "value"), is("yes")); + } + } + + @Nested + @DisplayName("Nested Tag Integration") + class NestedIntegration { + + @Test + @DisplayName("Resolves nested expression and !insert tags within value fields") + void resolvesNestedTags() throws IOException { + String yaml = """ + variables: + name: "World" + templates: + snippet: "Inserted Content" + test: !if + if: true + then: "Hello, ${name}!" + other: !if + if: true + then: !insert snippet + """; + + Map data = loadYaml(yaml); + + assertThat("Nested substitution inside !if should resolve", getNestedValue(data, "test"), + is("Hello, World!")); + assertThat("Nested !insert inside !if should resolve", getNestedValue(data, "other"), + is("Inserted Content")); + } + + @Test + @DisplayName("Should not resolve tags in inactive branches (Short-circuiting)") + void shortCircuitsInactiveBranches() throws IOException { + // We use a file that doesn't exist to prove the !include never runs + String yaml = """ + test: !if + if: true + then: "active" + else: !include non_existent_file.yaml + """; + + // If this throws an exception, the short-circuiting failed! + Map data = loadYaml(yaml); + + assertThat("The active branch should be the one resolved", data.get("test"), is("active")); + assertThat("The inactive branch should not trigger any warnings", logSession.getTrackedWarnings(), + not(hasItem(containsString("Failed to process !include")))); + } + + @Test + @DisplayName("Resolves deeply nested !if tags") + void resolvesDeeplyNestedIf() throws IOException { + String yaml = """ + test: !if + if: true + then: !if + if: true + then: "deep" + """; + + Map data = loadYaml(yaml); + + assertThat("Recursive resolution should handle nested !if tags", getNestedValue(data, "test"), + is("deep")); + } + + @Test + @DisplayName("Strips off null items in a list when !if is false and no else is provided") + @SuppressWarnings("null") + void stripsNullInSequence() throws IOException { + String yaml = """ + list: + - item1 + - !if + if: false + then: "item2" + - item3 + """; + + Map data = loadYaml(yaml); + List list = (List) data.get("list"); + + assertThat("The null result should be stripped, reducing list size", list.size(), is(2)); + assertThat("The first item should be preserved", list.get(0), is("item1")); + assertThat("The third item should shift to index 1", list.get(1), is("item3")); + } + + @Test + @DisplayName("Does not strip items in a list when !if resolves to a non-null value") + @SuppressWarnings("null") + void preservesElseInSequence() throws IOException { + String yaml = """ + list: + - !if + if: false + then: "no" + else: "yes" + """; + + Map data = loadYaml(yaml); + List list = (List) data.get("list"); + + assertThat("The list size should be 1", list.size(), is(1)); + assertThat("The item should resolve to the else value", list.get(0), is("yes")); + } + } + } + + @Nested + @DisplayName("Include Tag Specification") + class IncludeTests { + + @Nested + @DisplayName("Syntax & Argument Validation") + class SyntaxValidation { + + @ParameterizedTest(name = "Input [{0}] should warn about missing file parameter") + @ValueSource(strings = { "!include", "!include ''", "!include {}", "!include { file: null }" }) + void warnsOnMalformedInclude(String input) throws IOException { + Path yamlFile = writeFixture("includeTest.yaml", "a: " + input); + loadFixture(yamlFile); + + assertThat(logSession.getTrackedWarnings(), + hasItem(containsString("Failed to process !include: missing 'file' parameter"))); + } + + @Test + @DisplayName("Supports simple scalar syntax (!include file.yaml)") + void supportsSimpleScalarArgument() throws IOException { + writeFixture("simple.inc.yaml", "key: value"); + Path main = writeFixture("main.yaml", "toplevel: !include simple.inc.yaml"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "toplevel", "key"), equalTo("value")); + } + + @Test + @DisplayName("Supports URL-style scalar syntax (!include file.yaml?arg=value&bool)") + void supportsUrlStyleScalarArgument() throws IOException { + writeFixture("simple.inc.yaml", """ + key: ${arg1} + bool: ${bool} + """); + Path main = writeFixture("main.yaml", "toplevel: !include simple.inc.yaml?arg1=value&bool"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "toplevel", "key"), equalTo("value")); + assertThat(getNestedValue(data, "toplevel", "bool"), is(true)); + } + + @Test + @DisplayName("Supports map syntax with 'file' key (!include { file: ... })") + void supportsMapArgument() throws IOException { + writeFixture("included.inc.yaml", "key: value"); + Path main = writeFixture("main.yaml", """ + toplevel: !include + file: included.inc.yaml + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "toplevel", "key"), equalTo("value")); + } + } + + @Nested + @DisplayName("File Resolution & Recursion") + class Recursion { + + @Test + @DisplayName("Warns and continues when the included file does not exist") + void warnsWhenIncludeFileNotFound() throws IOException { + Path main = writeFixture("main.yaml", """ + data: !include missing.yaml + other: value + """); + + Map data = loadFixture(main); + + assertThat(logSession.getTrackedWarnings(), hasItem(allOf(containsString("Failed to process !include"), + containsString("missing.yaml"), containsString("No such file")))); + + assertThat("The rest of the document should still be processed", getNestedValue(data, "other"), + is("value")); + } + + @Test + @DisplayName("Supports deeply nested inclusions (chained files)") + void supportsDeeplyNestedIncludes() throws IOException { + Path main = writeFixture("main.yaml", "toplevel: !include level1.inc.yaml"); + writeFixture("level1.inc.yaml", "level1: !include level2.inc.yaml"); + writeFixture("level2.inc.yaml", "level2: leaf_value"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "toplevel", "level1", "level2"), equalTo("leaf_value")); + } + + @Test + @DisplayName("Detects and warns about circular inclusion loops to prevent stack overflow") + void preventsInfiniteLoopOnCircularInclusion() throws IOException { + Path main = writeFixture("a.yaml", "data: !include b.yaml"); + writeFixture("b.yaml", "data: !include c.yaml"); + writeFixture("c.yaml", "data: !include a.yaml"); + loadFixture(main); + + assertThat(logSession.getTrackedWarnings(), hasItem(containsString("Circular inclusion detected"))); + } + } + + @Nested + @DisplayName("Variable Scoping & Inheritance") + class VariableScoping { + + @Test + @DisplayName("Inherits variables defined in the parent context into the included file") + void inheritsVariablesFromParentContext() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + parent_var: "visible" + data: !include included.inc.yaml + """); + writeFixture("included.inc.yaml", "result: ${parent_var}"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "data", "result"), equalTo("visible")); + } + + @Test + @DisplayName("Propagates global variables through multiple nested include levels") + void propagatesGlobalVariablesThroughMultipleIncludeLevels() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + top_var: "hello" + root: !include mid.inc.yaml + """); + writeFixture("mid.inc.yaml", "mid_key: !include leaf.inc.yaml"); + writeFixture("leaf.inc.yaml", "leaf_key: ${top_var}"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "root", "mid_key", "leaf_key"), equalTo("hello")); + } + + @Test + @DisplayName("Global variables (main file) take precedence over variables in included files") + void prefersGlobalVariablesOverLocalVariables() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + target: "global" + data: !include + file: included.inc.yaml + """); + writeFixture("included.inc.yaml", """ + variables: + target: "local" + result: ${target} + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "data", "result"), equalTo("global")); + } + + @Test + @DisplayName("Allows overriding parent/global variables using the 'vars' argument in the !include tag") + void overridesParentVariablesUsingVarsArgument() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + target: "original" + data: !include + file: included.inc.yaml + vars: + target: "overridden" + """); + writeFixture("included.inc.yaml", "result: ${target}"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "data", "result"), equalTo("overridden")); + } + + @Test + @DisplayName("VARS exists in the included file context and can be used to reference the entire variable set passed to the include") + void varsKeywordReferencesEntireVariableSet() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + var1: "value1" + data: !include + file: included.inc.yaml + vars: + var2: "value2" + """); + writeFixture("included.inc.yaml", """ + variables: + local: "local_value" + + var1: ${VARS.var1} + var2: ${VARS.var2} + var3: ${VARS.local} + """); + Map data = loadFixture(main); + assertThat(getNestedValue(data, "data", "var1"), equalTo("value1")); + assertThat(getNestedValue(data, "data", "var2"), equalTo("value2")); + assertThat(getNestedValue(data, "data", "var3"), equalTo("local_value")); + } + } + + @Nested + @DisplayName("Path Resolution Strategy") + class PathResolution { + + @Test + @DisplayName("Resolves nested includes relative to the directory of the currently processing file") + void resolvesRelativePathsCorrectly() throws IOException { + Path main = writeFixture("main.yaml", "toplevel: !include scripts/level1.inc.yaml"); + + // level1 is in /scripts/, so it should find 'utils/level2' relative to itself + writeFixture("scripts/level1.inc.yaml", "data: !include utils/level2.inc.yaml"); + writeFixture("scripts/utils/level2.inc.yaml", "status: 'relative_success'"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "toplevel", "data", "status"), equalTo("relative_success")); + } + + @Test + @DisplayName("Resolves absolute path includes outside the base directory") + void resolvesAbsolutePathsCorrectly(@TempDir Path tempDir) throws IOException { + assertThat(tempDir, not(equalTo(sharedTempDir))); + Path includeFile = tempDir.resolve("level1.inc.yaml"); + assertTrue(includeFile.isAbsolute()); + Path main = writeFixture("main.yaml", "toplevel: !include " + includeFile); + writeFixture(includeFile.toString(), "data: absolute_include"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "toplevel", "data"), equalTo("absolute_include")); + } + + @Test + @DisplayName("Supports parent directory navigation using '..' in file paths") + void supportsParentDirectoryNavigation() throws IOException { + Path main = writeFixture("nested/main.yaml", "cfg: !include ../config.inc.yaml"); + writeFixture("config.inc.yaml", "version: 1.0"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "cfg", "version"), equalTo(1.0)); + } + + @Test + @DisplayName("Resolves '@' placeholder at start of include path to OPENHAB_CONF") + void resolvesAtPlaceholderInIncludePath() throws IOException { + writeFixture("test/include.inc.yaml", "key: value"); + Path main = writeFixture("main_at.yaml", "toplevel: !include '@test/include.inc.yaml'"); + + Path tempDir = Objects.requireNonNull(sharedTempDir); + try { + ComposerConfig.setRootsForTesting(tempDir, tempDir); + Map data = loadFixture(main); + assertThat(getNestedValue(data, "toplevel", "key"), is("value")); + } finally { + ComposerConfig.resetRootsForTesting(); + } + } + + @Test + @DisplayName("Resolves '$' placeholder to OPENHAB_CONF/yamlcomposer") + void resolvesDollarPlaceholderInIncludePath() throws IOException { + String topLevel = "yamlcomposer"; + writeFixture(topLevel + "/room/include.inc.yaml", "key: value"); + Path main = writeFixture(topLevel + "/room/main.yaml", "toplevel: !include '$room/include.inc.yaml'"); + + Path tempDir = Objects.requireNonNull(sharedTempDir); + try { + ComposerConfig.setRootsForTesting(tempDir, tempDir); + Map data = loadFixture(main); + assertThat(getNestedValue(data, "toplevel", "key"), is("value")); + } finally { + ComposerConfig.resetRootsForTesting(); + } + } + } + } + + @Nested + @DisplayName("Insert Tag Specification") + class InsertTests { + + @Nested + @DisplayName("Template Resolution") + class Resolution { + + @Test + @DisplayName("Inserts content defined in the top-level templates node") + void templateLookupWorks() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + simple: + scalar: "bar" + + target: !insert + template: simple + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "target", "scalar"), equalTo("bar")); + } + + @Test + @DisplayName("Templates node can be dynamically generated via substitutions and still be resolved") + void dynamicTemplatesNodeSupported() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + tplname: "my_template" + value: "foo" + + templates: + ${tplname}: ${value} + + data: !insert my_template + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "data"), equalTo("foo")); + } + + @Test + @DisplayName("Warns when the requested template key is missing") + void missingTemplateWarns() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + exists: "foo" + + target: !insert + template: does_not_exist + """); + + loadFixture(main); + + assertThat(logSession.getTrackedWarnings(), hasItem(containsString("template not found"))); + } + } + + @Nested + @DisplayName("Syntax & Argument Validation") + class SyntaxValidation { + + @ParameterizedTest(name = "Input [{0}] should warn about missing template parameter") + @ValueSource(strings = { "!insert", "!insert ''", "!insert {}", "!insert { template: null }" }) + void warnsOnMalformedInclude(String input) throws IOException { + String yaml = """ + templates: + valid: "foo" + + target: %s + """.formatted(input); + loadYaml(yaml); + + assertThat(logSession.getTrackedWarnings(), hasItem( + allOf(containsString("Failed to process !insert"), containsString("missing template name")))); + } + + @Test + @DisplayName("Supports simple scalar syntax (!insert template_name)") + void supportsSimpleScalarArgument() throws IOException { + String yaml = """ + templates: + simple: + key: value + toplevel: !insert simple + """; + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "toplevel", "key"), equalTo("value")); + } + + @Test + @DisplayName("Supports URL-style scalar syntax (!insert template_name?arg=value&bool)") + void supportsUrlStyleScalarArgument() throws IOException { + String yaml = """ + templates: + simple: + key: ${arg1} + bool: ${bool} + toplevel: !insert simple?arg1=value&bool + """; + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "toplevel", "key"), equalTo("value")); + assertThat(getNestedValue(data, "toplevel", "bool"), is(true)); + } + + @Test + @DisplayName("Supports map syntax with 'template' key (!insert { template: ... })") + void supportsMapArgument() throws IOException { + String yaml = """ + templates: + included: + key: value + toplevel: !insert + template: included + """; + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "toplevel", "key"), equalTo("value")); + } + } + + @Nested + @DisplayName("Scoping & Visibility") + class Scoping { + + @Test + @DisplayName("Templates resolve global variables by default") + void templateResolvesGlobalVariables() throws IOException { + String yaml = """ + variables: + global_val: "from_global" + + templates: + tpl: ${global_val} + + target: !insert + template: tpl + """; + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "target"), equalTo("from_global")); + } + + @Test + @DisplayName("Local 'vars' in !insert override global variables") + void insertVarsOverrideMainVars() throws IOException { + String yaml = """ + variables: + foo: "global_bar" + + templates: + vartpl: ${foo} + + target: !insert + template: vartpl + vars: + foo: "overridden" + """; + + Map data = loadYaml(yaml); + + assertThat(getNestedValue(data, "target"), equalTo("overridden")); + } + + @Test + @DisplayName("Insert works within an include file using that file's local templates") + void insertWithinIncludeWorks() throws IOException { + Path main = writeFixture("main.yaml", "content: !include child.yaml"); + + writeFixture("child.yaml", """ + templates: + local_tpl: "local_value" + + data: !insert + template: local_tpl + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "content", "data"), equalTo("local_value")); + } + + @Test + @DisplayName("Templates are file-local and not shared across boundaries") + void templatesAreFileLocal() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + parent_tpl: "secret" + content: !include child.yaml + """); + + writeFixture("child.yaml", """ + data: !insert + template: parent_tpl + """); + + loadFixture(main); + + assertThat(logSession.getTrackedWarnings(), hasItem(containsString("template not found"))); + } + } + } + + @Nested + @DisplayName("Packaging Specification") + class PackagingTests { + + @Nested + @DisplayName("Integration Styles") + class IntegrationStyles { + + @Test + @DisplayName("Packages can be defined using external include files") + void packageInclusionWorks() throws IOException { + // The Blueprint + writeFixture("package.inc.yaml", """ + things: + ${name}: + label: ${thing_label} + items: + ${name}: + label: ${item_label} + """); + + // The Main Composition + Path main = writeFixture("main.yaml", """ + packages: + basic1: !include + file: package.inc.yaml + vars: + name: basic1 + thing_label: "B1 Thing" + item_label: "B1 Item" + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "things", "basic1", "label"), equalTo("B1 Thing")); + assertThat(getNestedValue(data, "items", "basic1", "label"), equalTo("B1 Item")); + } + + @Test + @DisplayName("Packages can be defined using local templates") + void packageTemplateWorks() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + pkg_tpl: + things: + ${name}: + label: "Template Label" + + packages: + basic1: !insert + template: pkg_tpl + vars: + name: "from_tpl" + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "things", "from_tpl", "label"), equalTo("Template Label")); + } + + @Test + @DisplayName("Non-package content remains after merge") + void nonPackageContentRemainsAfterMerge() throws IOException { + Path main = writeFixture("main.yaml", """ + packages: + pkg1: !include { file: package.inc.yaml, vars: { name: "p1" } } + things: + static_thing: + label: "keep_me" + """); + writeFixture("package.inc.yaml", "things: { '${name}': { label: 'pkg' } }"); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "things", "static_thing", "label"), equalTo("keep_me")); + assertThat(getNestedValue(data, "things", "p1", "label"), equalTo("pkg")); + } + } + + @Nested + @DisplayName("Package ID Injection") + class PackageID { + + @Test + @DisplayName("Injects package ID into mapping-form include") + void packageIdInjectedIntoMappingInclude() throws IOException { + writeFixture("package.yaml", """ + result: + ${package_id}: 'active' + """); + Path main = writeFixture("main.yaml", """ + packages: + test_id: !include + file: package.yaml + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "result", "test_id"), equalTo("active")); + } + + @Test + @DisplayName("Injects package ID into scalar-form include") + void packageIdInjectedIntoScalarInclude() throws IOException { + writeFixture("package.yaml", """ + result: + ${package_id}: 'active' + """); + Path main = writeFixture("main.yaml", """ + packages: + test_id: !include package.yaml + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "result", "test_id"), equalTo("active")); + } + + @Test + @DisplayName("Injects package ID into mapping-form insert using named template") + void packageIdInjectedIntoMappingInsert() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + test_template: + result: + ${package_id}: 'active' + + packages: + test_id: !insert + template: test_template + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "result", "test_id"), equalTo("active")); + } + + @Test + @DisplayName("Injects package ID into scalar-form insert using named template") + void packageIdInjectedIntoScalarInsert() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + test_template: + result: + ${package_id}: 'active' + + packages: + test_id: !insert test_template + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "result", "test_id"), equalTo("active")); + } + + @Test + @DisplayName("Manual package ID variable takes precedence over automatic injection") + void packageIdIsOverridable() throws IOException { + writeFixture("package.yaml", "value: ${package_id}"); + + Path main = writeFixture("main.yaml", """ + packages: + default_id: !include + file: package.yaml + vars: + package_id: "custom_id" + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "value"), equalTo("custom_id")); + } + } + + @Nested + @DisplayName("Merge Strategies") + class MergeStrategies { + + @Test + @SuppressWarnings("unchecked") + @DisplayName("Deeply merges maps: overwrites shared scalars, appends lists, and preserves unique keys from both") + void defaultDeepMergeLogic() throws IOException { + writeFixture("pkg.yaml", """ + things: + thing: + scalar: package + config: + scalar1: package + scalar2: package + map1: + scalar1: package + scalar2: package + list1: + - package + """); + + Path main = writeFixture("main.yaml", """ + packages: + p1: !include pkg.yaml + things: + thing: + main_only_scalar: "preserved" + config: + scalar2: main + map1: + scalar2: main + scalar3: "main_only" + map2: + new_key: "main_only" + list1: + - main + """); + + Map data = loadFixture(main); + Map thing = (Map) getNestedValue(data, "things", + "thing"); + + // 1. Verify Overwrites (Main wins) + assertThat(getNestedValue(thing, "config", "scalar2"), equalTo("main")); + assertThat(getNestedValue(thing, "config", "map1", "scalar2"), equalTo("main")); + + // 2. Verify Package-only values are preserved + assertThat(getNestedValue(thing, "scalar"), equalTo("package")); + assertThat(getNestedValue(thing, "config", "scalar1"), equalTo("package")); + assertThat(getNestedValue(thing, "config", "map1", "scalar1"), equalTo("package")); + + // 3. Verify Main-only values are preserved (The Union) + assertThat(getNestedValue(thing, "main_only_scalar"), equalTo("preserved")); + assertThat(getNestedValue(thing, "config", "map1", "scalar3"), equalTo("main_only")); + assertThat(getNestedValue(thing, "config", "map2", "new_key"), equalTo("main_only")); + + // 4. Verify List behavior (Append) + assertThat(getNestedValue(thing, "config", "list1"), equalTo(List.of("package", "main"))); + } + + @Nested + @DisplayName("PackageMergeHelpers") + class PackageMergeHelpers { + + @Test + @DisplayName("Nested package include can override items using !replace and !remove") + void replaceAndRemoveInsideIncludedPackageSourceWork() throws IOException { + writeFixture("nested-items.inc.yaml", """ + things: + thing: + label: from_nested_package + config: + tags: + - from_nested + details: + source: nested + remove_me: true + items: + target_item: + label: base_label + tags: + - from_nested + metadata: + stateDescription: + value: from_nested + category: + value: from_nested + untouched_item: + label: untouched_from_nested + """); + + writeFixture("pkg-with-overrides.inc.yaml", """ + packages: + nested: !include nested-items.inc.yaml + items: + target_item: + tags: !replace + - from_outer + metadata: + stateDescription: !remove + category: !replace + value: from_outer + config: + origin: nested_package_override + """); + + Path main = writeFixture("main.yaml", """ + packages: + p1: !include pkg-with-overrides.inc.yaml + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "items", "target_item", "tags"), equalTo(List.of("from_outer"))); + assertThat((Map) getNestedValue(data, "items", "target_item", "metadata"), + not(hasKey("stateDescription"))); + assertThat(getNestedValue(data, "items", "target_item", "metadata", "category", "value"), + equalTo("from_outer")); + assertThat(getNestedValue(data, "items", "target_item", "metadata", "category", "config", "origin"), + equalTo("nested_package_override")); + assertThat(getNestedValue(data, "items", "untouched_item", "label"), + equalTo("untouched_from_nested")); + } + + @Test + @DisplayName("!remove directive deletes specific keys from the merge result") + void removeDirectiveWorks() throws IOException { + writeFixture("pkg.yaml", """ + things: + thing: + label: to_remove + scalar: package + config: + scalar1: package + map1: { key: val } + whole_thing_removed: { key: val } + thing_to_keep: { status: 'safe' } + """); + + Path main = writeFixture("main.yaml", """ + packages: + p1: !include pkg.yaml + things: + thing: + label: !remove + config: + map1: !remove + whole_thing_removed: !remove + """); + + Map data = loadFixture(main); + @SuppressWarnings("unchecked") + Map thing = (Map) getNestedValue(data, "things", + "thing"); + + // 1. Verify Removals + assertThat(thing, not(hasKey("label"))); + assertThat((Map) getNestedValue(thing, "config"), not(hasKey("map1"))); + assertThat((Map) getNestedValue(data, "things"), not(hasKey("whole_thing_removed"))); + + // 2. Verify Survival of Neighbors + assertThat(getNestedValue(thing, "scalar"), equalTo("package")); + assertThat(getNestedValue(thing, "config", "scalar1"), equalTo("package")); + assertThat(getNestedValue(data, "things", "thing_to_keep", "status"), equalTo("safe")); + } + + @Test + @DisplayName("!replace directive overwrites complex nodes instead of merging them") + void replaceDirectiveWorks() throws IOException { + writeFixture("pkg.yaml", """ + things: + thing: + map1: + scalar1: package + scalar2: package + list1: + - package + scalar_to_keep: package + """); + + Path main = writeFixture("main.yaml", """ + packages: + p1: !include pkg.yaml + things: + thing: + map1: !replace + scalar1: main + list1: !replace + - main + """); + + Map data = loadFixture(main); + @SuppressWarnings("unchecked") + Map thing = (Map) getNestedValue(data, "things", + "thing"); + + // !replace results in ONLY main's data + assertThat(getNestedValue(thing, "map1"), equalTo(Map.of("scalar1", "main"))); + assertThat(getNestedValue(thing, "list1"), equalTo(List.of("main"))); + assertThat(getNestedValue(thing, "scalar_to_keep"), equalTo("package")); + } + + @Test + @DisplayName("!replace and !remove inside !insert templates are applied only after package merge") + void insertTemplatePackageOverridesAreDeferredUntilOverridePhase() throws IOException { + writeFixture("pkg.yaml", """ + things: + thing: + map1: + from_package: true + remove_me: from_package + """); + + Path main = writeFixture("main.yaml", """ + templates: + overrides_tpl: + thing: + map1: !replace + from_main: true + remove_me: !remove + + packages: + p1: !include pkg.yaml + + things: !insert overrides_tpl + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "things", "thing", "map1"), equalTo(Map.of("from_main", true))); + assertThat((Map) getNestedValue(data, "things", "thing"), not(hasKey("remove_me"))); + } + + @Nested + @DisplayName("Defensive Fallbacks (Outside Package Context)") + class DefensiveFallbacks { + + @Test + @DisplayName("!remove in Map: Should self-delete even without a merge") + void removeMapSafety() throws IOException { + Path main = writeFixture("main_only.yaml", """ + config: + active: true + junk: !remove + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "config"), is(instanceOf(Map.class))); + assertThat((Map) getNestedValue(data, "config"), not(hasKey("junk"))); + assertThat(getNestedValue(data, "config", "active"), is(true)); + } + + @Test + @DisplayName("!remove in List: Should purge the entry even without a merge") + void removeListSafety() throws IOException { + Path main = writeFixture("list_only.yaml", """ + items: + - "A" + - !remove + - "B" + """); + + Map data = loadFixture(main); + List items = (List) data.get("items"); + + assertThat(items, contains("A", "B")); + assertThat(items, hasSize(2)); + } + + @Test + @DisplayName("!replace: Should unwrap to raw content to avoid 'Opaque Tag' artifacts") + void replaceSafety() throws IOException { + Path main = writeFixture("replace_only.yaml", """ + settings: + mode: !replace "standalone" + options: !replace + speed: fast + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "settings", "mode"), equalTo("standalone")); + assertThat(getNestedValue(data, "settings", "options", "speed"), equalTo("fast")); + } + + @Test + @DisplayName("Replace with insert works") + void replaceWithInsert() throws IOException { + Path main = writeFixture("replace_only.yaml", """ + templates: + test: bar + + settings: + mode: !replace + foo: !insert test + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "settings", "mode", "foo"), equalTo("bar")); + } + + @Test + @DisplayName("Replace with include works") + void replaceWithInclude() throws IOException { + writeFixture("test.yaml", "bar"); + Path main = writeFixture("replace_only.yaml", """ + templates: + test: bar + + settings: + mode: !replace + foo: !include test.yaml + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "settings", "mode", "foo"), equalTo("bar")); + } + } + } + } + + @Nested + @DisplayName("Conflict Resolution") + class Conflicts { + + @Test + @DisplayName("Earlier packages take precedence over later packages in the sequence") + void earlierPackageWinsConflict() throws IOException { + // Package A: The first one processed + writeFixture("pkg_a.inc.yaml", """ + things: + shared_id: + status: "from_a" + only_in_a: "value_a" + """); + + // Package B: The second one processed + writeFixture("pkg_b.inc.yaml", """ + things: + shared_id: + status: "from_b" + only_in_b: "value_b" + """); + + Path main = writeFixture("main.yaml", """ + packages: + instance_1: !include pkg_a.inc.yaml + instance_2: !include pkg_b.inc.yaml + """); + + Map data = loadFixture(main); + + // 1. Precedence Check: 'from_a' should remain because it was merged into Main + // first + assertThat(getNestedValue(data, "things", "shared_id", "status"), equalTo("from_a")); + + // 2. Union Check: Maps are still merged, so unique keys from B are still added + assertThat(getNestedValue(data, "things", "shared_id", "only_in_a"), equalTo("value_a")); + assertThat(getNestedValue(data, "things", "shared_id", "only_in_b"), equalTo("value_b")); + } + } + } + + @Nested + @DisplayName("Universal Structural Tags (Outside Package Context)") + class UniversalStructuralTagTests { + + @Test + @DisplayName("!replace: Fallback to raw content for all YAML types (Map, List, Scalar)") + @SuppressWarnings("null") + void replaceFallbackForAllTypes() throws IOException { + String yaml = """ + map_context: + key: !replace { a: b } + list_context: + key: !replace [ item1, item2 ] + scalar_context: + key: !replace "just a string" + """; + + Map data = loadYaml(yaml); + + // Verify Map preservation: should treat as standard Map + Object mapNode = getNestedValue(data, "map_context", "key"); + assertThat(mapNode, instanceOf(Map.class)); + assertThat(((Map) mapNode).get("a"), is("b")); + + // Verify List preservation: should treat as standard List + Object listNode = getNestedValue(data, "list_context", "key"); + assertThat(listNode, instanceOf(List.class)); + assertThat((List) listNode, contains("item1", "item2")); + + // Verify Scalar preservation: should treat as standard String + Object scalarNode = getNestedValue(data, "scalar_context", "key"); + assertThat(scalarNode, is("just a string")); + } + + @Test + @DisplayName("!remove: Map key removal (Primary Use Case)") + void removeFunctionsGloballyInMaps() throws IOException { + String yaml = """ + target: + victim_key: !remove "delete me" + safe_key: "keep me" + """; + + Map data = loadYaml(yaml); + @SuppressWarnings("unchecked") + Map target = (Map) data.get("target"); + + assertThat("The specified key should be removed from the map", target, not(hasKey("victim_key"))); + assertThat("Unrelated keys must be preserved", target, hasEntry("safe_key", "keep me")); + } + + @Test + @DisplayName("!remove: List item removal (The 'Filter' Use Case)") + void removeFunctionsGloballyInLists() throws IOException { + // If !remove is inside a list, the intuition is that it removes + // the item it references from the list. + String yaml = """ + my_list: + - "item 1" + - "item 2" + - !remove + """; + + Map data = loadYaml(yaml); + @SuppressWarnings("unchecked") + List list = (List) Objects.requireNonNull(data.get("my_list")); + + // The result should only contain "item 2" + assertThat("The list should be filtered by the !remove tag", list, contains("item 1", "item 2")); + assertThat("The list size should reflect the removal", list.size(), is(2)); + } + } + + @Nested + @DisplayName("Merge Key (<<) Specifications") + class MergeKeyTests { + + /** + * Ensures our custom {@code RecursiveTransformer} implementation adheres to the + * official YAML 1.1 Merge Key Language-Independent Type specification. + *

+ * Because we manually handle {@code <<} to support dynamic tags like {@code !if} + * and {@code !include}, we must strictly replicate the "First-Key-Wins" + * precedence rules to avoid breaking standard YAML behavior. + * * @see YAML 1.1 Merge Key Spec + */ + @Nested + @DisplayName("Standard YAML 1.1 Compliance") + class StandardCompliance { + + @Test + @DisplayName("Spec: Local keys override merged keys") + void localKeyPrecedence() throws IOException { + String yaml = """ + base: &base + status: "original" + type: "base" + target: + status: "overridden" + <<: *base + """; + + Map data = loadYaml(yaml); + // Local 'status' exists first, so 'original' is ignored. + assertThat(getNestedValue(data, "target", "status"), equalTo("overridden")); + assertThat(getNestedValue(data, "target", "type"), equalTo("base")); + } + + @Test + @DisplayName("Spec: Sequence Merge - Earlier mappings override later ones") + void sequenceMergePrecedence() throws IOException { + String yaml = """ + m1: &m1 { val: "first" } + m2: &m2 { val: "second" } + target: + <<: [*m1, *m2] + """; + + Map data = loadYaml(yaml); + // m1 is processed first; val is set to "first". + // When m2 is processed, val already exists, so "second" is skipped. + assertThat(getNestedValue(data, "target", "val"), equalTo("first")); + } + + @Test + @DisplayName("Spec: Multiple merge keys - First merge key wins") + void multipleMergeKeysFirstWins() throws IOException { + String yaml = """ + m1: &m1 { common: "from_m1", unique_a: 1 } + m2: &m2 { common: "from_m2", unique_b: 2 } + target: + <<: *m1 + <<: *m2 + """; + + Map data = loadYaml(yaml); + + // Following the "unless the key already exists" rule: + // 1. unique_a and common ("from_m1") are inserted. + // 2. common already exists, so "from_m2" is ignored. + // 3. unique_b is inserted. + assertThat(getNestedValue(data, "target", "common"), equalTo("from_m1")); + assertThat(getNestedValue(data, "target", "unique_a"), equalTo(1)); + assertThat(getNestedValue(data, "target", "unique_b"), equalTo(2)); + } + + @Test + @DisplayName("Spec: Complex precedence (Local > Merge1 > Merge2)") + void complexPrecedence() throws IOException { + String yaml = """ + m1: &m1 { a: "m1", b: "m1", c: "m1" } + m2: &m2 { a: "m2", b: "m2", d: "m2" } + target: + a: "local" + <<: *m1 + <<: *m2 + """; + + Map data = loadYaml(yaml); + + assertAll(() -> assertThat("Local wins", getNestedValue(data, "target", "a"), equalTo("local")), + () -> assertThat("First merge wins", getNestedValue(data, "target", "b"), equalTo("m1")), + () -> assertThat("From first merge", getNestedValue(data, "target", "c"), equalTo("m1")), + () -> assertThat("From second merge", getNestedValue(data, "target", "d"), equalTo("m2"))); + } + + @Test + @DisplayName("Spec: Deep merge is NOT supported") + void mergeIsNotRecursive() throws IOException { + String yaml = """ + default_meta: &default_meta + tags: { level: "info", persistent: true } + target: + <<: *default_meta + tags: { level: "debug" } + """; + + Map data = loadYaml(yaml); + // The local 'tags' key blocks the entire merged 'tags' map. + assertThat(getNestedValue(data, "target", "tags", "level"), equalTo("debug")); + assertThat(getNestedValue(data, "target", "tags", "persistent"), is(nullValue())); + } + } + + @Nested + @DisplayName("Alias Support") + class AliasSupport { + + @Test + @DisplayName("Merge with simple alias mapping") + void mergeWithAliasMapping() throws IOException { + String yaml = """ + base: &base + x: 1 + y: 2 + a: [foo, bar] + target: + z: 3 + <<: *base + """; + + Map data = loadYaml(yaml); + assertThat(getNestedValue(data, "target", "x"), equalTo(1)); + assertThat(getNestedValue(data, "target", "y"), equalTo(2)); + assertThat(getNestedValue(data, "target", "a"), equalTo(List.of("foo", "bar"))); + assertThat(getNestedValue(data, "target", "z"), equalTo(3)); + } + + @Test + @DisplayName("Merge with alias sequence of mappings") + void mergeWithAliasSequenceOfMappings() throws IOException { + String yaml = """ + m1: &m1 { a: 10 } + m2: &m2 { b: 20 } + target: + <<: [*m1, *m2] + """; + + Map data = loadYaml(yaml); + assertThat(getNestedValue(data, "target", "a"), equalTo(10)); + assertThat(getNestedValue(data, "target", "b"), equalTo(20)); + } + + @Test + @DisplayName("Merge with alias referencing another alias") + void mergeWithAliasReferencingAlias() throws IOException { + String yaml = """ + base1: &base1 { foo: 1 } + base2: &base2 { <<: *base1 } + target: + <<: *base2 + """; + + Map data = loadYaml(yaml); + assertThat(getNestedValue(data, "target", "foo"), equalTo(1)); + } + } + + @Nested + @DisplayName("Conditional Integration (!if)") + class ConditionalIntegration { + + @Test + @DisplayName("Merge with !if resolving to a mapping (truthy)") + void mergeWithTruthyIf() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + is_prod: true + prod_settings: { timeout: 30, retry: 3 } + target: + existing: "value" + <<: !if + if: ${is_prod} + then: ${prod_settings} + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "target", "timeout"), equalTo(30)); + assertThat(getNestedValue(data, "target", "existing"), equalTo("value")); + } + + @Test + @DisplayName("Merge with !if resolving to alternative mapping (else branch)") + void mergeWithFalsyIfWithElseBranch() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + env: "dev" + target: + <<: !if + if: "${env == 'prod'}" + then: { color: "red" } + else: { color: "green" } + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "target", "color"), equalTo("green")); + } + + @Test + @DisplayName("Merge with !if with falsy condition without else results in a no-op") + @SuppressWarnings("null") + void mergeWithFalsyIfWithoutElse() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + is_prod: false + target: + existing: "preserved" + <<: !if + if: ${is_prod} + then: { secret: "password" } + """); + + Map data = loadFixture(main); + @SuppressWarnings("unchecked") + Map target = (Map) data.get("target"); + + assertThat(target, equalTo(Map.of("existing", "preserved"))); + + // No warnings should be emitted for this valid use case + assertThat(logSession.getTrackedWarnings(), is(empty())); + } + + @Test + @DisplayName("Merge with sequence form !if (multiple branches)") + void mergeWithSequenceIf() throws IOException { + Path main = writeFixture("main.yaml", """ + target: + <<: !if + - if: false + then: { branch: 1 } + - elseif: true + then: { branch: 2 } + - else: { branch: 3 } + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "target", "branch"), equalTo(2)); + } + } + + @Nested + @DisplayName("Inclusion Strategy") + class Inclusion { + + @Test + @DisplayName("Merge with !include in scalar form") + void mergeWithScalarInclude() throws IOException { + writeFixture("inc.yaml", "foo: include1"); + Path main = writeFixture("main.yaml", """ + simple: + <<: !include inc.yaml + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "simple", "foo"), equalTo("include1")); + } + + @Test + @DisplayName("Merge with !include in mapping form with variables") + void mergeWithMappingInclude() throws IOException { + writeFixture("inc.yaml", "greeting: 'Hello ${name}'"); + Path main = writeFixture("main.yaml", """ + simple: + <<: !include + file: inc.yaml + vars: + name: "World" + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "simple", "greeting"), equalTo("Hello World")); + } + + @Test + @DisplayName("Merge with !include using substitutions in filename and vars") + void mergeWithIncludeSubstitutions() throws IOException { + writeFixture("production.yaml", """ + mode: prod + owner: ${owner} + """); + Path main = writeFixture("main.yaml", """ + variables: + env: production + user: gemini + + target: + <<: !include + file: "${env}.yaml" + vars: + owner: "${user}" + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "target", "mode"), equalTo("prod")); + assertThat(getNestedValue(data, "target", "owner"), equalTo("gemini")); + } + + @Test + @DisplayName("Merge with !include in a map") + void mergeWithIncludeInMap() throws IOException { + writeFixture("inc.yaml", "foo: include1"); + Path main = writeFixture("main.yaml", """ + simple: + <<: + bar: !include inc.yaml + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "simple", "bar", "foo"), equalTo("include1")); + } + + @Test + @DisplayName("Merge with !include non-existent file results in a no-op") + void mergeWithNonExistentInclude() throws IOException { + Path main = writeFixture("main.yaml", "simple: { <<: !include missing.yaml }"); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "simple"), equalTo(Map.of())); + assertThat(logSession.getTrackedWarnings(), not(hasItem(containsString("Expected a mapping")))); + } + } + + @Nested + @DisplayName("Substitution and Inheritance") + class Substitution { + + @Test + @DisplayName("Merge keys work with substitutions for dynamic mapping injection") + void mergeWithSubstitution() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + map1: + foo: bar + baz: "${foo}" + simple: + <<: ${map1} + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "simple", "foo"), equalTo("bar")); + assertThat(getNestedValue(data, "simple", "baz"), nullValue()); + } + + @Test + @DisplayName("Merge keys handle undefined variables as no-ops") + void mergeWithUndefinedVariable() throws IOException { + Path main = writeFixture("main.yaml", """ + target: + existing_key: "preserved" + <<: ${undefined_var} + """); + + Map data = loadFixture(main); + @SuppressWarnings("unchecked") + Map target = (Map) Objects + .requireNonNull(getNestedValue(data, "target")); + + assertThat(target, hasEntry("existing_key", "preserved")); + assertThat(target.size(), equalTo(1)); + + assertThat(logSession.getTrackedWarnings(), not(hasItem(containsString("Expected a mapping")))); + } + + @Test + @DisplayName("Merge keys support lists of substitutions") + void mergeWithSubstitutionsThatReturnsListOfMaps() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + map1: { foo: bar } + map2: { qux: quux } + array_merge: + <<: ${[map1, map2]} + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "array_merge", "foo"), equalTo("bar")); + assertThat(getNestedValue(data, "array_merge", "qux"), equalTo("quux")); + } + + @Test + @DisplayName("!literal on parent allows internal substitutions on merge keys to process") + void mergeWithParentLiteral() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + map1: { foo: bar } + parent_literal: !literal + <<: !sub ${map1} + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "parent_literal", "foo"), equalTo("bar")); + } + + @Test + @DisplayName("Merge keys work with substitution inside a map") + void mergeWithSubstitutionInMap() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + map1: + foo: bar + baz: "${foo}" + simple: + <<: + qux: ${map1} + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "simple", "qux", "foo"), equalTo("bar")); + assertThat(getNestedValue(data, "simple", "qux", "baz"), nullValue()); + } + + @Test + @DisplayName("Merge keys work with substitution inside a list") + void mergeWithSubstitutionInList() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + map1: + foo: bar + baz: "${foo}" + simple: + <<: + qux: + - ${map1} + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "simple", "qux"), equalTo(List.of(Map.of("foo", "bar")))); + } + + @Test + @DisplayName("Conditional logic in substitutions without else results in a no-op merge") + void mergeWithConditionalWithoutElse() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + empty_map: {} + conditionally_empty: + <<: ${empty_map if false} + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "conditionally_empty"), equalTo(Map.of())); + + assertThat(logSession.getTrackedWarnings(), is(empty())); + } + } + + @Nested + @DisplayName("Templates and Collections") + class TemplatesAndCollections { + + @Test + @DisplayName("Merge keys work with !insert templates (scalar form)") + void mergeWithScalarTemplates() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + base: { foo: bar } + simple: + <<: !insert base + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "simple", "foo"), equalTo("bar")); + } + + @Test + @DisplayName("Merge keys work with !insert templates (mapping form)") + void mergeWithMappingTemplate() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + device: + type: ${dev_type} + vendor: "openHAB" + simple: + <<: !insert + template: device + vars: + dev_type: Light + """); + + Map data = loadFixture(main); + assertThat(getNestedValue(data, "simple", "type"), equalTo("Light")); + assertThat(getNestedValue(data, "simple", "vendor"), equalTo("openHAB")); + } + + @Test + @DisplayName("Merge keys with !include at template top-level (placeholder-valued merge source)") + void mergeKeysWithIncludeAtTemplateTopLevel() throws IOException { + writeFixture("base.yaml", """ + shared_attr: from_base + """); + + Path main = writeFixture("main.yaml", """ + templates: + merged_template: + <<: !include base.yaml + own_attr: from_template + device: !insert merged_template + """); + + Map data = loadFixture(main); + + // merged_template should contain both attributes. + assertThat(getNestedValue(data, "device", "shared_attr"), equalTo("from_base")); + assertThat(getNestedValue(data, "device", "own_attr"), equalTo("from_template")); + } + + @Test + @DisplayName("Merge keys with !insert at template top-level (placeholder-valued merge source)") + void mergeKeysWithInsertAtTemplateTopLevel() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + base_data: + a: 1 + b: 2 + extended_template: + <<: !insert base_data + c: 3 + result: !insert extended_template + """); + + Map data = loadFixture(main); + + // Control case: merge key inside a template value with !insert source. + assertThat(getNestedValue(data, "result", "a"), equalTo(1)); + assertThat(getNestedValue(data, "result", "b"), equalTo(2)); + assertThat(getNestedValue(data, "result", "c"), equalTo(3)); + } + + @Test + @DisplayName("Merge keys at templates TOP-LEVEL with !include placeholder (demonstrates the bug)") + void mergeKeysAtTemplatesMapLevel() throws IOException { + writeFixture("base_templates.yaml", """ + base_template: + value: from_base + """); + + Path main = writeFixture("main.yaml", """ + templates: + <<: !include base_templates.yaml + merged_template: !insert base_template + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "merged_template", "value"), equalTo("from_base")); + } + } + + @Test + @DisplayName("Merge keys in !include vars are visible to included file") + void mergeKeysInIncludeVars() throws IOException { + writeFixture("include.inc.yaml", "foo: ${foo}"); + + Path main = writeFixture("main.yaml", """ + packages: + foo_package: !include + file: include.inc.yaml + vars: + <<: + foo: bar + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "foo"), equalTo("bar")); + } + + @Nested + @DisplayName("Type Fidelity") + class TypeFidelity { + + @Test + @DisplayName("Merge keys retain primitive types (int, boolean) from !include") + void mergeRetainsTypesFromInclude() throws IOException { + // inc.yaml contains actual boolean and int types + writeFixture("inc.yaml", """ + active: true + timeout: 30 + """); + Path main = writeFixture("main.yaml", """ + device: + <<: !include inc.yaml + """); + + Map data = loadFixture(main); + Object active = getNestedValue(data, "device", "active"); + Object timeout = getNestedValue(data, "device", "timeout"); + + // Verify values AND types + assertThat(active, allOf(equalTo(true), instanceOf(Boolean.class))); + assertThat(timeout, allOf(equalTo(30), instanceOf(Integer.class))); + } + + @Test + @DisplayName("Merge keys retain types from substitution map injection") + void mergeRetainsTypesFromSub() throws IOException { + Path main = writeFixture("main.yaml", """ + variables: + settings: + enabled: true + port: 8080 + target: + <<: ${settings} + """); + + Map data = loadFixture(main); + Object enabled = getNestedValue(data, "target", "enabled"); + Object port = getNestedValue(data, "target", "port"); + + assertThat(enabled, instanceOf(Boolean.class)); + assertThat(port, instanceOf(Integer.class)); + } + + @Test + @DisplayName("Merge keys retain types when using !insert with variable overrides") + void mergeRetainsTypesFromInsert() throws IOException { + Path main = writeFixture("main.yaml", """ + templates: + base: + value: ${in_val} + target: + <<: !insert + template: base + vars: + in_val: 100 + """); + + Map data = loadFixture(main); + Object value = getNestedValue(data, "target", "value"); + + // If bug exists, value would be "100" (String) instead of 100 (Integer) + assertThat(value, allOf(equalTo(100), instanceOf(Integer.class))); + } + + @Test + @DisplayName("Merge keys retain nested List and Map structures") + void mergeRetainsComplexStructures() throws IOException { + writeFixture("inc.yaml", """ + tags: [a, b, c] + config: { level: 1 } + """); + Path main = writeFixture("main.yaml", """ + target: + <<: !include inc.yaml + """); + + Map data = loadFixture(main); + + assertThat(getNestedValue(data, "target", "tags"), instanceOf(List.class)); + assertThat(getNestedValue(data, "target", "config"), instanceOf(Map.class)); + assertThat(getNestedValue(data, "target", "config", "level"), instanceOf(Integer.class)); + } + } + } + + @Nested + @DisplayName("File Cache Behavior") + class FileCacheTests { + @Test + @DisplayName("Caches include bytes and mtime on first load") + void cachesIncludeEntry() throws Exception { + Path included = writeFixture("cache_included.inc.yaml", "key: value"); + Path main = writeFixture("main_cache.yaml", """ + data1: !include cache_included.inc.yaml + data2: !include cache_included.inc.yaml + """); + + ConcurrentHashMap includeCache = new ConcurrentHashMap<>(); + YamlComposer.load(main, p -> { + }, logSession, includeCache); + + Path real = included.toRealPath(); + assertTrue(includeCache.containsKey(real)); + CacheEntry entry = includeCache.get(real); + assertNotNull(entry); + assertArrayEquals(Files.readAllBytes(real), entry.bytes()); + assertEquals(Files.getLastModifiedTime(real).toMillis(), entry.mtime()); + } + + @Test + @DisplayName("Refreshes cached entry when mtime differs") + void refreshesCacheOnMtimeChange() throws Exception { + Path included = writeFixture("cache_refresh.inc.yaml", "a: old"); + Path main = writeFixture("main_refresh.yaml", "data: !include cache_refresh.inc.yaml"); + + Path real = included.toRealPath(); + + ConcurrentHashMap includeCache = new ConcurrentHashMap<>(); + // Insert a stale entry with an older mtime and different bytes + long staleMtime = Math.max(0L, Files.getLastModifiedTime(real).toMillis() - 10_000L); + includeCache.put(real, new CacheEntry("stale".getBytes(), staleMtime)); + + // Update the file to ensure a newer mtime and new content + Files.writeString(included, "a: refreshed"); + + YamlComposer.load(main, p -> { + }, logSession, includeCache); + + CacheEntry entry = includeCache.get(real); + assertNotNull(entry); + assertArrayEquals(Files.readAllBytes(real), entry.bytes()); + assertEquals(Files.getLastModifiedTime(real).toMillis(), entry.mtime()); + } + } + + /** + * Load a YAML fixture file from the test resources. + *

+ * This helper method simplifies loading fixture files by automatically resolving the path + * relative to the standard test resources directory and parsing the YAML content. + *

+ * The method also includes enhanced error handling to provide clear context about + * which test and fixture caused a failure, making it easier to diagnose issues + * when a fixture cannot be loaded or parsed correctly. + *

+ * The returned Map is guaranteed to contain no Placeholder instances, as the method will + * fail the test if any unresolved placeholders are found in the loaded data. + * This ensures that all placeholders are properly resolved before the test continues. + * + * @param source the name of the YAML file to load (relative to the fixture directory) + * @return the parsed YAML content as a Map + * @throws IOException if an error occurs reading the file + */ + @SuppressWarnings({ "unchecked", "null" }) + private Map loadFixture(Path source) throws IOException { + // If 'source' is absolute (like a temp file), resolve() returns 'source' as is. + // If it's relative, it appends it to SOURCE_PATH. + Path filePath = source.isAbsolute() ? source : SOURCE_PATH.resolve(source); + + try { + ConcurrentHashMap includeCache = new ConcurrentHashMap<>(); + Object result = YamlComposer.load(filePath, path -> { + }, logSession, includeCache); + + if (result instanceof Map dataMap) { + Map map = (Map) dataMap; + assertNoPlaceholders(map); + return map; + } + + fail("Fixture did not produce a Map structure: " + source); + } catch (Exception e) { + fail("\n%s#%s: %s\nError loading fixture '%s': %s".formatted( + currentTest.getTestClass().get().getSimpleName(), currentTest.getTestMethod().get().getName(), + currentTest.getDisplayName(), source, e.getMessage()), e); + } + return Map.of(); + } + + private Map loadYaml(String content) throws IOException { + Path tempFile = Objects.requireNonNull(sharedTempDir).resolve("inline-test-" + System.nanoTime() + ".yaml"); + Files.writeString(tempFile, content); + return loadFixture(tempFile); + } + + private @Nullable Object loadYamlValue(String content) throws IOException { + Path tempFile = Objects.requireNonNull(sharedTempDir).resolve("inline-test-" + System.nanoTime() + ".yaml"); + Files.writeString(tempFile, content); + + ConcurrentHashMap includeCache = new ConcurrentHashMap<>(); + Object result = YamlComposer.load(tempFile, path -> { + }, logSession, includeCache); + + if (result != null) { + assertNoPlaceholders(result); + } + return result; + } + + /** + * Recursively assert that the given object graph contains no Placeholder + * instances. Uses identity-based cycle detection to avoid infinite loops on + * self-referencing structures. + */ + private void assertNoPlaceholders(Object root) { + Objects.requireNonNull(root); + Set visited = Collections.newSetFromMap(new IdentityHashMap<>()); + ArrayDeque path = new ArrayDeque<>(); + + checkForPlaceholders(root, visited, path); + } + + private void checkForPlaceholders(Object obj, Set visited, ArrayDeque path) { + if (obj == null || obj instanceof Number || obj instanceof Boolean) { + return; + } + + if (obj instanceof String str) { + // Check for instances where String.valueOf or toString was called on a Placeholder, + // which would indicate an unresolved placeholder that was coerced to a string during processing + if (str.matches("(?s)^[A-Z].+Placeholder\\[.*")) { + failAt(path, "unresolved Placeholder string: " + str); + } + return; + } + + if (!visited.add(obj)) { + return; + } + + if (obj instanceof Placeholder) { + failAt(path, "unresolved Placeholder instance: " + obj.getClass().getName()); + } + + if (obj instanceof Map m) { + for (Map.Entry e : m.entrySet()) { + String keyName = String.valueOf(e.getKey()); + path.addLast(keyName); + checkForPlaceholders(e.getKey(), visited, path); // Check the key itself + checkForPlaceholders(e.getValue(), visited, path); // Check the value + path.removeLast(); + } + } else if (obj instanceof Iterable it) { + int i = 0; + for (Object o : it) { + path.addLast("[" + (i++) + "]"); + checkForPlaceholders(o, visited, path); + path.removeLast(); + } + } else if (obj.getClass().isArray()) { + int len = Array.getLength(obj); + for (int i = 0; i < len; i++) { + path.addLast("[" + i + "]"); + checkForPlaceholders(Array.get(obj, i), visited, path); + path.removeLast(); + } + } + } + + private void failAt(ArrayDeque path, String reason) { + TestInfo testInfo = Objects.requireNonNull(currentTest); + String p = path.isEmpty() ? "" : " at " + String.join("/", path); + fail("\n%s#%s: %s\nFound %s%s".formatted(testInfo.getTestClass().get().getSimpleName(), + testInfo.getTestMethod().get().getName(), testInfo.getDisplayName(), reason, p)); + } + + /** + * Writes a helper YAML file to the shared temporary directory. + * + * @param fileName The name of the file (e.g., + * "subOverIncludeLiterals.inc.yaml") + * @param content The YAML content for the include file + */ + protected Path writeFixture(String fileName, String content) throws IOException { + Path inputPath = Path.of(fileName); + + Path includePath = Objects.requireNonNull(sharedTempDir).resolve(inputPath).normalize(); + + // Create parent directories if the filename contains a subfolder (e.g., + // "includes/file.yaml") + Path parentDir = includePath.getParent(); + if (parentDir != null && !Files.exists(parentDir)) { + Files.createDirectories(parentDir); + } + + Files.writeString(includePath, content); + return includePath; + } + + /** + * Retrieve a nested value from a map using the provided keys. + *

+ * This method navigates through a map structure using the given keys in order. + * If a key is not found or the current value is not a map, the method returns + * {@code null}. + * + * @param data the map to retrieve the value from; must not be {@code null}. + * @param key the sequence of keys to navigate the map; must not be + * {@code null}. + * @return the nested value if found, or {@code null} if a key is missing or the + * value is not a map. + */ + private static @Nullable Object getNestedValue(Map data, String... key) { + if (data == null) { + return null; + } + Object value = data; + for (String k : key) { + if (value instanceof Map map) { + value = map.get(k); + } else { + return null; + } + } + return value; + } +} diff --git a/bundles/pom.xml b/bundles/pom.xml index fdc5390854..0877946966 100644 --- a/bundles/pom.xml +++ b/bundles/pom.xml @@ -33,6 +33,7 @@ org.openhab.io.metrics org.openhab.io.neeo org.openhab.io.openhabcloud + org.openhab.io.yamlcomposer org.openhab.transform.basicprofiles org.openhab.transform.bin2json diff --git a/itests/org.openhab.io.yamlcomposer.tests/NOTICE b/itests/org.openhab.io.yamlcomposer.tests/NOTICE new file mode 100644 index 0000000000..38d625e349 --- /dev/null +++ b/itests/org.openhab.io.yamlcomposer.tests/NOTICE @@ -0,0 +1,13 @@ +This content is produced and maintained by the openHAB project. + +* Project home: https://www.openhab.org + +== Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License 2.0 which is available at +https://www.eclipse.org/legal/epl-2.0/. + +== Source Code + +https://github.com/openhab/openhab-addons diff --git a/itests/org.openhab.io.yamlcomposer.tests/itest.bndrun b/itests/org.openhab.io.yamlcomposer.tests/itest.bndrun new file mode 100644 index 0000000000..a83f004767 --- /dev/null +++ b/itests/org.openhab.io.yamlcomposer.tests/itest.bndrun @@ -0,0 +1,76 @@ +-include: ../itest-common.bndrun + +Bundle-SymbolicName: ${project.artifactId} +Fragment-Host: org.openhab.io.yamlcomposer + +-runrequires: \ + bnd.identity;id='org.openhab.io.yamlcomposer.tests',\ + bnd.identity;id='org.openhab.core',\ + bnd.identity;id='org.openhab.io.yamlcomposer' + +# We would like to use the "volatile" storage only +-runblacklist: \ + bnd.identity;id='org.openhab.core.storage.json' + +# +# done +# +-runbundles: \ + biz.aQute.tester.junit-platform;version='[7.2.3,7.2.4)',\ + ch.qos.logback.classic;version='[1.5.32,1.5.33)',\ + ch.qos.logback.core;version='[1.5.32,1.5.33)',\ + com.fasterxml.jackson.core.jackson-annotations;version='[2.21.0,2.21.1)',\ + com.fasterxml.jackson.core.jackson-core;version='[2.21.3,2.21.4)',\ + com.fasterxml.jackson.core.jackson-databind;version='[2.21.3,2.21.4)',\ + com.fasterxml.jackson.dataformat.jackson-dataformat-yaml;version='[2.21.3,2.21.4)',\ + com.google.gson;version='[2.13.2,2.13.3)',\ + com.google.guava;version='[33.6.0,33.6.1)',\ + com.google.guava.failureaccess;version='[1.0.3,1.0.4)',\ + com.sun.jna;version='[5.18.1,5.18.2)',\ + com.sun.xml.bind.jaxb-osgi;version='[2.3.9,2.3.10)',\ + io.methvin.directory-watcher;version='[0.19.1,0.19.2)',\ + jakarta.annotation-api;version='[2.1.1,2.1.2)',\ + jakarta.inject.jakarta.inject-api;version='[2.0.1,2.0.2)',\ + jakarta.xml.bind-api;version='[2.3.3,2.3.4)',\ + javax.measure.unit-api;version='[2.2.0,2.2.1)',\ + junit-jupiter-api;version='[5.14.4,5.14.5)',\ + junit-jupiter-engine;version='[5.14.4,5.14.5)',\ + junit-platform-commons;version='[1.14.4,1.14.5)',\ + junit-platform-engine;version='[1.14.4,1.14.5)',\ + junit-platform-launcher;version='[1.14.4,1.14.5)',\ + org.apache.aries.spifly.dynamic.bundle;version='[1.3.7,1.3.8)',\ + org.apache.felix.configadmin;version='[1.9.26,1.9.27)',\ + org.apache.felix.http.servlet-api;version='[1.2.0,1.2.1)',\ + org.apache.felix.scr;version='[2.2.18,2.2.19)',\ + org.apache.servicemix.specs.activation-api-1.2.1;version='[1.2.1,1.2.2)',\ + org.eclipse.equinox.event;version='[1.7.300,1.7.301)',\ + org.eclipse.jetty.http;version='[9.4.58,9.4.59)',\ + org.eclipse.jetty.io;version='[9.4.58,9.4.59)',\ + org.eclipse.jetty.security;version='[9.4.58,9.4.59)',\ + org.eclipse.jetty.server;version='[9.4.58,9.4.59)',\ + org.eclipse.jetty.servlet;version='[9.4.58,9.4.59)',\ + org.eclipse.jetty.util;version='[9.4.58,9.4.59)',\ + org.eclipse.jetty.util.ajax;version='[9.4.58,9.4.59)',\ + org.glassfish.hk2.osgi-resource-locator;version='[3.0.0,3.0.1)',\ + org.hamcrest;version='[3.0.0,3.0.1)',\ + org.objectweb.asm;version='[9.9.1,9.9.2)',\ + org.objectweb.asm.commons;version='[9.9.1,9.9.2)',\ + org.objectweb.asm.tree;version='[9.9.1,9.9.2)',\ + org.objectweb.asm.tree.analysis;version='[9.9.1,9.9.2)',\ + org.objectweb.asm.util;version='[9.9.1,9.9.2)',\ + org.openhab.core;version='[5.2.0,5.2.1)',\ + org.openhab.core.test;version='[5.2.0,5.2.1)',\ + org.openhab.io.yamlcomposer;version='[5.2.0,5.2.1)',\ + org.openhab.io.yamlcomposer.tests;version='[5.2.0,5.2.1)',\ + org.opentest4j;version='[1.3.0,1.3.1)',\ + org.ops4j.pax.logging.pax-logging-api;version='[2.3.3,2.3.4)',\ + org.osgi.service.component;version='[1.5.1,1.5.2)',\ + org.osgi.service.event;version='[1.4.1,1.4.2)',\ + org.osgi.util.function;version='[1.2.0,1.2.1)',\ + org.osgi.util.promise;version='[1.3.0,1.3.1)',\ + org.snakeyaml.engine;version='[3.0.1,3.0.2)',\ + org.yaml.snakeyaml;version='[2.5.0,2.5.1)',\ + si-quantity;version='[2.2.3,2.2.4)',\ + si-units;version='[2.2.3,2.2.4)',\ + tech.units.indriya;version='[2.2.3,2.2.4)',\ + uom-lib-common;version='[2.2.0,2.2.1)' \ No newline at end of file diff --git a/itests/org.openhab.io.yamlcomposer.tests/pom.xml b/itests/org.openhab.io.yamlcomposer.tests/pom.xml new file mode 100644 index 0000000000..af9758e8d5 --- /dev/null +++ b/itests/org.openhab.io.yamlcomposer.tests/pom.xml @@ -0,0 +1,25 @@ + + + + 4.0.0 + + + org.openhab.addons.itests + org.openhab.addons.reactor.itests + 5.2.0-SNAPSHOT + + + org.openhab.io.yamlcomposer.tests + + openHAB Add-ons :: Integration Tests :: YAML Composer IO Tests + + + + org.openhab.addons.bundles + org.openhab.io.yamlcomposer + ${project.version} + + + + diff --git a/itests/org.openhab.io.yamlcomposer.tests/src/main/java/org/openhab/io/yamlcomposer/internal/YamlComposerWatchServiceOSGiTest.java b/itests/org.openhab.io.yamlcomposer.tests/src/main/java/org/openhab/io/yamlcomposer/internal/YamlComposerWatchServiceOSGiTest.java new file mode 100644 index 0000000000..c7f6783379 --- /dev/null +++ b/itests/org.openhab.io.yamlcomposer.tests/src/main/java/org/openhab/io/yamlcomposer/internal/YamlComposerWatchServiceOSGiTest.java @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2010-2026 Contributors to the openHAB project + * + * See the NOTICE file(s) distributed with this work for additional + * information. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.openhab.io.yamlcomposer.internal; + +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.openhab.core.service.WatchService.Kind; +import org.openhab.core.test.java.JavaOSGiTest; + +/** + * OSGi integration tests for {@link YamlComposerWatchService}. + * + * @author Jimmy Tanagra - Initial contribution + */ +public class YamlComposerWatchServiceOSGiTest extends JavaOSGiTest { + private YamlComposerWatchService watchService; + private Path testSourceDir; + private Path testOutputDir; + + @BeforeEach + public void setUp() throws IOException { + watchService = getService(YamlComposerWatchService.class); + assertNotNull(watchService); + + testSourceDir = ComposerConfig.sourceRoot(); + testOutputDir = ComposerConfig.outputRoot(); + + deleteRecursively(testSourceDir); + deleteRecursively(testOutputDir); + + Files.createDirectories(testSourceDir.resolve("defs")); + Files.createDirectories(testOutputDir); + } + + @AfterEach + public void tearDown() throws IOException { + deleteRecursively(testSourceDir); + deleteRecursively(testOutputDir); + } + + @Test + public void shouldGenerateOutputFromSourceAndRecompileOnIncludeChange() throws IOException { + Path includeFile = testSourceDir.resolve("defs/items.inc.yaml"); + Path mainFile = testSourceDir.resolve("main.yaml"); + Path generatedFile = testOutputDir.resolve("main.yaml"); + + Files.writeString(includeFile, """ + - type: Switch + name: MySwitch + label: Label One + """); + Files.writeString(mainFile, """ + version: 1 + items: !include defs/items.inc.yaml + """); + + watchService.processWatchEvent(Kind.CREATE, mainFile); + + waitForAssert(() -> { + assertTrue(Files.exists(generatedFile)); + assertTrue(fileContains(generatedFile, "Label One")); + }); + + Files.writeString(includeFile, """ + - type: Switch + name: MySwitch + label: Label Two + """); + + watchService.processWatchEvent(Kind.MODIFY, includeFile); + + waitForAssert(() -> { + assertTrue(Files.exists(generatedFile)); + assertTrue(fileContains(generatedFile, "Label Two")); + }); + } + + @Test + public void shouldRemoveGeneratedOutputOnSourceDelete() throws IOException { + Path mainFile = testSourceDir.resolve("delete-me.yaml"); + Path generatedFile = testOutputDir.resolve("delete-me.yaml"); + + Files.writeString(mainFile, """ + version: 1 + items: + - type: Switch + name: DeleteMe + """); + + watchService.processWatchEvent(Kind.CREATE, mainFile); + + waitForAssert(() -> assertTrue(Files.exists(generatedFile))); + + watchService.processWatchEvent(Kind.DELETE, mainFile); + + waitForAssert(() -> assertFalse(Files.exists(generatedFile))); + } + + private static boolean fileContains(Path file, String expectedContent) { + try { + return Files.readString(file).contains(expectedContent); + } catch (IOException e) { + throw new AssertionError("Failed to read file " + file, e); + } + } + + private static void deleteRecursively(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + + Files.walkFileTree(path, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.deleteIfExists(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.deleteIfExists(dir); + return FileVisitResult.CONTINUE; + } + }); + } +} diff --git a/itests/pom.xml b/itests/pom.xml index 9c88ac099f..95b5020f96 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -30,6 +30,7 @@ org.openhab.binding.modbus.tests org.openhab.binding.mqtt.ruuvigateway.tests org.openhab.binding.ntp.tests + org.openhab.io.yamlcomposer.tests org.openhab.binding.systeminfo.tests org.openhab.binding.tradfri.tests org.openhab.persistence.mapdb.tests diff --git a/tools/static-code-analysis/spotbugs/suppressions.xml b/tools/static-code-analysis/spotbugs/suppressions.xml index 01578efa69..0c4ea83e5b 100644 --- a/tools/static-code-analysis/spotbugs/suppressions.xml +++ b/tools/static-code-analysis/spotbugs/suppressions.xml @@ -37,6 +37,11 @@ + + + + +