From 40b990367b528972c41f826fa554b8dd4a415f47 Mon Sep 17 00:00:00 2001 From: Victor Moene Date: Tue, 24 Jun 2025 16:44:42 +0200 Subject: [PATCH] Added json promise type Signed-off-by: Victor Moene --- promise-types/json/README.md | 155 +++++++++++++++++ promise-types/json/enable.cf | 6 + promise-types/json/example.cf | 13 ++ promise-types/json/json_promise_type.py | 152 +++++++++++++++++ promise-types/json/test.cf | 161 ++++++++++++++++++ .../json/tests/to_append.expected.json | 22 +++ .../json/tests/to_modify.expected.json | 3 + promise-types/json/tests/to_modify.start.json | 3 + .../json/tests/to_overwrite.expected.json | 7 + .../json/tests/to_overwrite.start.json | 3 + 10 files changed, 525 insertions(+) create mode 100644 promise-types/json/README.md create mode 100644 promise-types/json/enable.cf create mode 100644 promise-types/json/example.cf create mode 100644 promise-types/json/json_promise_type.py create mode 100644 promise-types/json/test.cf create mode 100644 promise-types/json/tests/to_append.expected.json create mode 100644 promise-types/json/tests/to_modify.expected.json create mode 100644 promise-types/json/tests/to_modify.start.json create mode 100644 promise-types/json/tests/to_overwrite.expected.json create mode 100644 promise-types/json/tests/to_overwrite.start.json diff --git a/promise-types/json/README.md b/promise-types/json/README.md new file mode 100644 index 0000000..5d69c70 --- /dev/null +++ b/promise-types/json/README.md @@ -0,0 +1,155 @@ +Promise type for manipulating `json` files + +## Attributes + +| Name | Type | Description | +|---------------|-----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------| +| `object` | `data container` | json object type. It can also be json arrays | +| `array` | `slist`, `rlist`, `ilist`, `data array` | json array type. `slist`, `rlist` and `ilist` will only create string arrays. To create array of other types, use `data array` | +| `string` | `string` | json string type | +| `number` | `real`, `int` | json number type | +| `primitive` | `string` | Primitives are values that are either `"true"`, `"false"` or `"null"` in json | + +## Examples + +### Write to a whole file + +To write to a json file, you can do: + +```cfengine3 +bundle agent main +{ + json: + "/tmp/newfile.json" + array => '["hello", "world"]'; +} +``` + +The resulting `/tmp/newfile.json` will only contain the array: + +```json +["hello", "world"] +``` + +If the `/tmp/newfile.json` doesn't exist, it will be created. If it exists and contains some data, they will be overwritten. + +### Write to a specific field + +Given a json file `/tmp/oldfile.json`, +```json +{ + "foo": "bar" +} +``` + +we can modify/append a field by doing: + +```cfengine3 +bundle agent main +{ + json: + "/tmp/oldfile.json:greeting" + array => '["hello", "world"]'; +} +``` + +And the content of `/tmp/oldfile.json` will become: + +```json +{ + "foo": "bar", + "greeting": ["hello", "world"] +} +``` + +If the field doesn't exist, it is appended. If it already exists, its data will be overwritten. + +### Writing types + +In order to write compound type such as arrays containg booleans, numbers, etc... One has to use the `data container` type in the policy. + +To see what happens if we use + +```cfengine3 +bundle agent main +{ + vars: + "json_data" + data => '[1.2, true, "hello!"]'; + + "real_list" + rlist => {"1.2", "2.3"}; + "bool_list" + slist => {"true", "false"}; + + json: + "/tmp/example_1.json:json_data" + array => "$(json_data)"; + + "/tmp/example_2.json:real_list" + array => "$(real_list)"; + "/tmp/example_2.json:bool_list" + array => "$(bool_list)"; +} +``` + +We can compare the content of `/tmp/example_1.json` and `/tmp/example_2.json`: + +```json +{ + "json_data": [1.2, true, "hello!"] +} +``` + +```json +{ + "real_list": ["1.2", "2.3"], + "bool_list": ["true", "false"] +} +``` + +As we can see, using slist, rlist or ilist to write arrays will always result in array of strings. If we want more complex arrays using containg number, true, false or null, then we need to use the `data container` type. + +## Not implemented yet + +The copy attribute allows to copy the content of a json file into another json file. For example, `/tmp/oldfile.json` contains the following: + +```json +{ + "hello": "world" +} +``` + +We can copy it into the `/tmp/newfile.json` in the field `"oldfile"` by doing: + +```cfengine3 +bundle agent main +{ + json: + "/tmp/newfile.json:oldfile" + copy => "/tmp/oldfile.json"; +} +``` + +```json +{ + "oldfile": { + "hello": "world" + } +} +``` + + +## Authors + +This software was created by the team at [Northern.tech](https://northern.tech), with many contributions from the community. +Thanks everyone! + +## Contribute + +Feel free to open pull requests to expand this documentation, add features, or fix problems. +You can also pick up an existing task or file an issue in [our bug tracker](https://northerntech.atlassian.net/). + +## License + +This software is licensed under the MIT License. See LICENSE in the root of the repository for the full license text. diff --git a/promise-types/json/enable.cf b/promise-types/json/enable.cf new file mode 100644 index 0000000..cccfe94 --- /dev/null +++ b/promise-types/json/enable.cf @@ -0,0 +1,6 @@ +promise agent json +# @brief Define json promise type +{ + path => "$(sys.workdir)/modules/promises/json_promise_type.py"; + interpreter => "/usr/bin/python3"; +} diff --git a/promise-types/json/example.cf b/promise-types/json/example.cf new file mode 100644 index 0000000..91d3a84 --- /dev/null +++ b/promise-types/json/example.cf @@ -0,0 +1,13 @@ +promise agent json +# @brief Define json promise type +{ + path => "$(sys.workdir)/modules/promises/json_promise_type.py"; + interpreter => "/usr/bin/python3"; +} + +bundle agent main +{ + json: + "/tmp/myusers.json:name" + string => "John" +} diff --git a/promise-types/json/json_promise_type.py b/promise-types/json/json_promise_type.py new file mode 100644 index 0000000..95f1969 --- /dev/null +++ b/promise-types/json/json_promise_type.py @@ -0,0 +1,152 @@ +import os +import json + +from cfengine import PromiseModule, ValidationError, Result, AttributeObject + + +def is_number(num): + try: + float(num) + return True + except ValueError: + return False + + +def is_json_serializable(string): + try: + json.loads(string) + return True + except json.JSONDecodeError: + return False + + +class JsonPromiseTypeModule(PromiseModule): + + def __init__(self, **kwargs): + super(JsonPromiseTypeModule, self).__init__( + name="json_promise_module", version="0.0.1", **kwargs + ) + + self.types = ["object", "array", "string", "number", "primitive"] + self.valid_attributes = ( + self.types + ) # for now, the only valid attributes are the types. + + def create_attribute_object(self, attributes): + data = {t: None for t in self.valid_attributes} + for attr, val in attributes.items(): + data[attr] = val + return AttributeObject(data) + + def validate_promise(self, promiser, attributes, metadata): + + for attr in attributes: + if attr not in self.valid_attributes: + raise ValidationError("Unknown attribute '{}'".format(attr)) + + present_types = [t for t in self.types if t in attributes] + if present_types == 0: + raise ValidationError( + "The promiser '{}' is missing a type attribute. The possible types are {}".format( + promiser, str(self.types) + ) + ) + elif len(present_types) > 1: + raise ValidationError( + "The attributes {} cannot be together".format(str(self.types)) + ) + + filename, _, _ = promiser.partition(":") + if os.path.exists(filename) and not os.path.isfile(filename): + raise ValidationError( + "'{}' already exists and is not a file".format(filename) + ) + + if not filename.endswith(".json"): + raise ValidationError("'{}' is not a json file") + + model = self.create_attribute_object(attributes) + if ( + model.object + and isinstance(model.object, str) + and not is_json_serializable(model.object) + ): + raise ValidationError( + "'{}' is not a valid data container".format(model.object) + ) + + if model.array: + if isinstance(model.array, str): + if not is_json_serializable(model.array): + raise ValidationError( + "'{}' is not a valid list".format(model.array) + ) + + if not isinstance(json.loads(model.array), list): + raise ValidationError( + "'{}' is not a valid data".format(model.array) + ) + + elif not isinstance(model.array, list): + raise ValidationError( + "'{}' is not a valid data array".format(model.array) + ) + + if model.number and not is_number(model.number): + raise ValidationError( + "'{}' is not a valid int or real".format(model.number) + ) + + if model.primitive and model.primitive not in ["true", "false", "null"]: + raise ValidationError( + "expected 'true', 'false' or 'null' but got '{}".format(model.primitive) + ) + + def evaluate_promise(self, promiser, attributes, metadata): + model = self.create_attribute_object(attributes) + filename, _, field = promiser.partition(":") + + # type conversion + + datatype = next(t for t in self.types if t in attributes) + + match datatype: + case "object" | "array": + data = ( + json.loads(attributes[datatype]) + if isinstance(attributes[datatype], str) + else attributes[datatype] + ) + case "number": + data = float(model.number) if "." in model.number else int(model.number) + case "primitive": + data = None if model.primitive == "null" else model.primitive == "true" + case _: # strings + data = attributes[datatype] + + # json manipulation + + try: + with open(filename, "r+") as f: + content = json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + content = {} + + if field: + if field in content and content[field] == data: + Result.KEPT + content[field] = data + else: + if content == data: + Result.KEPT + content = data + + with open(filename, "w") as f: + json.dump(content, f, indent=4) + + self.log_info("Updated '{}'".format(filename)) + Result.REPAIRED + + +if __name__ == "__main__": + JsonPromiseTypeModule().start() diff --git a/promise-types/json/test.cf b/promise-types/json/test.cf new file mode 100644 index 0000000..ecb5fca --- /dev/null +++ b/promise-types/json/test.cf @@ -0,0 +1,161 @@ +body common control +{ + inputs => { "$(sys.libdir)/stdlib.cf" }; + version => "1.0"; + bundlesequence => { "init", "test", "check", "cleanup"}; +} + +####################################################### + +bundle agent init +{ + vars: + "to_overwrite" + data => readjson("$(this.promise_dirname)/tests/to_overwrite.start.json", 100k); + "to_modify" + data => readjson("$(this.promise_dirname)/tests/to_modify.start.json", 100k); + + files: + "$(this.promise_dirname)/tests/to_overwrite.json" + create => "true", + content => "$(to_overwrite)"; + "$(this.promise_dirname)/tests/to_modify.json" + create => "true", + content => "$(to_modify)"; + "$(this.promise_dirname)/tests/to_append_1.json" + create => "true"; +} + +####################################################### + +promise agent json +{ + path => "$(this.promise_dirname)/json_promise_type.py"; + interpreter => "/usr/bin/python3"; +} + +bundle agent test +{ + vars: + "objects" + data => '{ "bar": [1, 2, 3] }'; + "int_arrays" + data => '[1,2,3]'; + "arrays" + ilist => { "1", "2" }; # slist == rlist == ilist -> lists of string + "numbers" + int => "1"; + "strings" + string => "hello"; + "bools" + string => "true"; + "nulls" + string => "null"; + + + json: + "$(this.promise_dirname)/tests/to_overwrite.json" + object => "@(objects)"; + + "$(this.promise_dirname)/tests/to_modify.json:Hello" + string => "$(strings)"; + + "$(this.promise_dirname)/tests/to_append_1.json:a" + object => "@(objects)"; + "$(this.promise_dirname)/tests/to_append_1.json:b" + object => "@(int_arrays)"; + "$(this.promise_dirname)/tests/to_append_1.json:c" + array => "@(int_arrays)"; + "$(this.promise_dirname)/tests/to_append_1.json:d" + string => "$(strings)"; + "$(this.promise_dirname)/tests/to_append_1.json:e" + number => "$(numbers)"; + "$(this.promise_dirname)/tests/to_append_1.json:f" + primitive => "$(bools)"; + "$(this.promise_dirname)/tests/to_append_1.json:g" + primitive => "$(nulls)"; + + "$(this.promise_dirname)/tests/to_append_2.json:a" + object => '{ "bar": [1, 2, 3] }'; + "$(this.promise_dirname)/tests/to_append_2.json:b" + object => '[1,2,3]'; + "$(this.promise_dirname)/tests/to_append_2.json:c" + array => '[1,2,3]'; + "$(this.promise_dirname)/tests/to_append_2.json:d" + string => "hello"; + "$(this.promise_dirname)/tests/to_append_2.json:e" + number => "1"; + "$(this.promise_dirname)/tests/to_append_2.json:f" + primitive => "true"; + "$(this.promise_dirname)/tests/to_append_2.json:g" + primitive => "null"; + +} + +####################################################### + +bundle agent check +{ + vars: + "to_overwrite_content" + data => readjson("$(this.promise_dirname)/tests/to_overwrite.json", 100k); + "to_modify_content" + data => readjson("$(this.promise_dirname)/tests/to_modify.json", 100k); + "to_append_1_content" + data => readjson("$(this.promise_dirname)/tests/to_append_1.json", 100k); + "to_append_2_content" + data => readjson("$(this.promise_dirname)/tests/to_append_2.json", 100k); + + "to_overwrite_content_true" + data => readjson("$(this.promise_dirname)/tests/to_overwrite.expected.json", 100k); + "to_modify_content_true" + data => readjson("$(this.promise_dirname)/tests/to_modify.expected.json", 100k); + "to_append_content_true" + data => readjson("$(this.promise_dirname)/tests/to_append.expected.json", 100k); + + "to_overwrite_content_indices" + slist => getindices("to_overwrite_content"); + "to_modify_content_indices" + slist => getindices("to_modify_content"); + "to_append_1_content_indices" + slist => getindices("to_append_1_content"); + "to_append_2_content_indices" + slist => getindices("to_append_2_content"); + + "to_overwrite_content_true_indices" + slist => getindices("to_overwrite_content_true"); + "to_modify_content_true_indices" + slist => getindices("to_modify_content_true"); + "to_append_content_true_indices" + slist => getindices("to_append_content_true"); + + classes: + "ok" + expression => and ( + strcmp("$(to_overwrite_content[$(to_overwrite_content_indices)])", "$(to_overwrite_content_true[$(to_overwrite_content_true_indices)])"), + strcmp("$(to_modify_content[$(to_modify_content_indices)])", "$(to_modify_content_true[$(to_modify_content_true_indices)])"), + strcmp("$(to_append_1_content[$(to_append_1_content_indices)])", "$(to_append_2_content[$(to_append_2_content_indices)])"), + strcmp("$(to_append_1_content[$(to_append_1_content_indices)])", "$(to_append_content_true[$(to_append_content_true_indices)])") + ); + + reports: + ok:: + "$(this.promise_filename) Pass"; + !ok:: + "$(this.promise_filename) FAIL"; +} + +# ####################################################### + +bundle agent cleanup +{ + files: + "$(this.promise_dirname)/tests/to_overwrite.json" + delete => tidy; + "$(this.promise_dirname)/tests/to_modify.json" + delete => tidy; + "$(this.promise_dirname)/tests/to_append_1.json" + delete => tidy; + "$(this.promise_dirname)/tests/to_append_2.json" + delete => tidy; +} diff --git a/promise-types/json/tests/to_append.expected.json b/promise-types/json/tests/to_append.expected.json new file mode 100644 index 0000000..8c84adc --- /dev/null +++ b/promise-types/json/tests/to_append.expected.json @@ -0,0 +1,22 @@ +{ + "a": { + "bar": [ + 1, + 2, + 3 + ] + }, + "b": [ + 1, + 2, + 3 + ], + "c": [ + "1", + "2" + ], + "d": "hello", + "e": 1, + "f": true, + "g": null +} diff --git a/promise-types/json/tests/to_modify.expected.json b/promise-types/json/tests/to_modify.expected.json new file mode 100644 index 0000000..1ded609 --- /dev/null +++ b/promise-types/json/tests/to_modify.expected.json @@ -0,0 +1,3 @@ +{ + "Hello": "hello" +} diff --git a/promise-types/json/tests/to_modify.start.json b/promise-types/json/tests/to_modify.start.json new file mode 100644 index 0000000..02bb47e --- /dev/null +++ b/promise-types/json/tests/to_modify.start.json @@ -0,0 +1,3 @@ +{ + "Hello": "World" +} diff --git a/promise-types/json/tests/to_overwrite.expected.json b/promise-types/json/tests/to_overwrite.expected.json new file mode 100644 index 0000000..7117d3f --- /dev/null +++ b/promise-types/json/tests/to_overwrite.expected.json @@ -0,0 +1,7 @@ +{ + "bar": [ + 1, + 2, + 3 + ] +} diff --git a/promise-types/json/tests/to_overwrite.start.json b/promise-types/json/tests/to_overwrite.start.json new file mode 100644 index 0000000..02bb47e --- /dev/null +++ b/promise-types/json/tests/to_overwrite.start.json @@ -0,0 +1,3 @@ +{ + "Hello": "World" +}