diff --git a/CHANGELOG.md b/CHANGELOG.md index 77ce03dd..d535653f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## [Version 1.4.1](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.4.1) - Feature release - 2025-11-26 +## [Version 1.4.2](https://github.com/dataiku/dss-plugin-pi-server/releases/tag/v1.4.2) - Feature release - 2025-11-26 - Add a AF hierarchy downloader diff --git a/custom-recipes/pi-system-af-tree/recipe.json b/custom-recipes/pi-system-af-tree/recipe.json index 5b917a43..05439d53 100644 --- a/custom-recipes/pi-system-af-tree/recipe.json +++ b/custom-recipes/pi-system-af-tree/recipe.json @@ -5,23 +5,14 @@ "icon": "icon-pi-system icon-cogs" }, "kind": "PYTHON", - "selectableFromDataset": "input_dataset", "paramsPythonSetup": "browse_af_tree.py", "inputRoles": [ - { - "name": "input_dataset", - "label": "Dataset containing paths or tags", - "description": "", - "arity": "UNARY", - "required": false, - "acceptsDataset": true - } ], "outputRoles": [ { "name": "api_output", - "label": "Main output displayed name", + "label": "Attributes dataset", "description": "", "arity": "UNARY", "required": true, diff --git a/custom-recipes/pi-system-af-tree/recipe.py b/custom-recipes/pi-system-af-tree/recipe.py index ed4c7fdd..0c74051b 100644 --- a/custom-recipes/pi-system-af-tree/recipe.py +++ b/custom-recipes/pi-system-af-tree/recipe.py @@ -1,5 +1,5 @@ import dataiku -from dataiku.customrecipe import get_input_names_for_role, get_recipe_config, get_output_names_for_role +from dataiku.customrecipe import get_recipe_config, get_output_names_for_role from safe_logger import SafeLogger from osisoft_plugin_common import ( get_credentials, PerformanceTimer @@ -34,7 +34,6 @@ def next_tree_item(tree_data): yield item -input_dataset = get_input_names_for_role('input_dataset') output_names_stats = get_output_names_for_role('api_output') config = get_recipe_config() tree_data = config.get("treeData", []) @@ -42,6 +41,7 @@ def next_tree_item(tree_data): logger.info("Initialization with config config={}".format(logger.filter_secrets(config))) auth_type, username, password, server_url, is_ssl_check_disabled = get_credentials(config) +is_ssl_check_disabled = config.get("is_ssl_check_disabled", False) # Because no advanced parameter switch network_timer = PerformanceTimer() processing_timer = PerformanceTimer() @@ -51,7 +51,7 @@ def next_tree_item(tree_data): schema = [ {'name': 'title', 'type': 'string'}, {'name': 'template_name', 'type': 'string'}, - {'name': 'category_names', 'type': 'array'}, + {'name': 'category_names', 'type': 'string'}, {'name': 'path', 'type': 'string'}, {'name': 'id', 'type': 'string'}, {'name': 'url', 'type': 'string'}, diff --git a/js/pi-system_treecontroller.js b/js/pi-system_treecontroller.js index 09c97d83..25ccea82 100644 --- a/js/pi-system_treecontroller.js +++ b/js/pi-system_treecontroller.js @@ -1,113 +1,219 @@ var app = angular.module('piSystemTreeApp.module', []); -app.service('TreeDataService', function() { +app.service('TreeDataService', function () { // This will store the shared tree data this.treeData = []; + this.templateTreeData = []; // Optional: helper methods - this.setTreeData = function(data) { + this.setTreeData = function (data) { this.treeData = data; }; - this.getTreeData = function() { + this.getTreeData = function () { return this.treeData; }; + + this.setTemplateTreeData = function (data) { + this.templateTreeData = data; + }; + + this.getTemplateTreeData = function () { + return this.templateTreeData; + }; }); app.controller('AfExplorerFormCtrl', [ - '$scope', - '$stateParams', - 'CodeMirrorSettingService', - 'TreeDataService', - function($scope, $stateParams, CodeMirrorSettingService, TreeDataService) { - + '$scope', + '$stateParams', + 'CodeMirrorSettingService', + 'TreeDataService', + function ($scope, $stateParams, CodeMirrorSettingService, TreeDataService) { + $scope.paramDesc = { 'parameterSetId': 'basic-auth', 'mandatory': true }; - + $scope.treeData = TreeDataService.getTreeData(); + $scope.templateTreeData = TreeDataService.getTemplateTreeData(); $scope.config.attributeList = $scope.config.attributeList || []; $scope.config.selectedAttributes = $scope.config.selectedAttributes || []; - + $scope.config.clickedNodes = $scope.config.clickedNodes || []; + $scope.editorOptions = CodeMirrorSettingService.get("text/plain"); - $scope.init = function() { - DataikuAPI.plugins.listAccessiblePresets('pi-system', $stateParams.projectKey, 'basic-auth').success(function (data) { - $scope.inlineParams = data.inlineParams; - $scope.inlinePluginParams = data.inlinePluginParams; - $scope.accessiblePresets = []; - if (data.definableInline) { - $scope.accessiblePresets.push({ - name:"INLINE", - label:"Manually defined", usable:true, - description: "Define values for these parameters" - }); - } - data.presets.forEach(function(p) { - $scope.accessiblePresets.push({name:"PRESET " + p.name, label:p.name, usable:p.usable, description:p.description}); - }); - $scope.accessibleParameterSetDescriptions = $scope.accessiblePresets.map(function(p) { - return p.description || 'No description'; - }); - }).error(setErrorInScope.bind($scope.errorScope)); + $scope.onAdvancedToggle = function () { + if (!$scope.config.show_advanced_parameters) { + $scope.config.is_ssl_check_disabled = false; + $scope.config.elements_max_count = null; + $scope.config.attributes_max_count = null; + } else { + if ($scope.config.elements_max_count === null || $scope.config.elements_max_count === undefined || $scope.config.elements_max_count === "") { + $scope.config.elements_max_count = 100; + } + if ($scope.config.attributes_max_count === null || $scope.config.attributes_max_count === undefined || $scope.config.attributes_max_count === "") { + $scope.config.attributes_max_count = 100; + } + } }; - - $scope.getServers = function(){ - $scope.callPythonDo({parameterName: "server_name"}).then(function(data){ + + $scope.init = function () { + $scope.config.show_advanced_parameters = $scope.config.show_advanced_parameters || false; + $scope.config.activeTab = $scope.config.activeTab || 'element'; + DataikuAPI.plugins.listAccessiblePresets('pi-system', $stateParams.projectKey, 'basic-auth').success(function (data) { + $scope.inlineParams = data.inlineParams; + $scope.inlinePluginParams = data.inlinePluginParams; + $scope.accessiblePresets = []; + if (data.definableInline) { + $scope.accessiblePresets.push({ + name: "INLINE", + label: "Manually defined", usable: true, + description: "Define values for these parameters" + }); + } + data.presets.forEach(function (p) { + $scope.accessiblePresets.push({ name: "PRESET " + p.name, label: p.name, usable: p.usable, description: p.description }); + }); + $scope.accessibleParameterSetDescriptions = $scope.accessiblePresets.map(function (p) { + return p.description || 'No description'; + }); + }).error(setErrorInScope.bind($scope.errorScope)); + if ($scope.authConfigured() === true) { + $scope.authSectionVisible = false; + $scope.showTreeData = true; + $scope.showTemplateTreeData = true; + } + $scope.config.template = $scope.config.template || "-- Any --"; + $scope.onAdvancedToggle(); + }; + + $scope.getServers = function () { + $scope.callPythonDo({ parameterName: "server_name" }).then(function (data) { $scope.server_name = data.choices; }); }; - $scope.getDatabases = function() { - $scope.callPythonDo({parameterName: "database_name"}).then(function(data){ + $scope.getDatabases = function () { + $scope.callPythonDo({ parameterName: "database_name" }).then(function (data) { $scope.database_name = data.choices; }); }; - - $scope.initializeTree = function(){ - console.log("initialization: "); - console.log($scope.config.treeData); - if (!$scope.config.treeData || $scope.config.treeData.length === 0){ - $scope.callPythonDo({method: "get_children_from_db", parent: $scope.config.database_name}).then(function(data){ + + $scope.authSectionVisible = $scope.authSectionVisible || true; + + $scope.toggleAuthSection = function () { + $scope.authSectionVisible = !$scope.authSectionVisible; + }; + + $scope.authConfigured = function () { + console.log('authConfigured check'); + return $scope.hasPreset() && $scope.config.database_name && $scope.config.database_name.length > 0 && $scope.config.server_name && $scope.config.server_name.length > 0; + } + $scope.explore = function () { + if ($scope.authConfigured()) { + $scope.showTreeData = true; + $scope.showTemplateTreeData = true; + } + }; + + $scope.hasPreset = function () { + return $scope.config.credentials && $scope.config.credentials.mode && $scope.config.credentials.mode !== 'NONE' && $scope.config.credentials.name + } + + $scope.cleanTree = function () { + $scope.config.treeData = []; + $scope.config.clickedNodes = []; + $scope.config.attributeList = []; + } + + $scope.initializeTree = function () { + console.log("initialization: "); + if (!$scope.config.treeData || $scope.config.treeData.length === 0) { + $scope.callPythonDo({ method: "get_children_from_db", parent: $scope.config.database_name }).then(function (data) { console.log("ALX:data2=" + JSON.stringify(data)); TreeDataService.setTreeData(data.choices); - $scope.config.treeData = TreeDataService.getTreeData(); - }); - } - }; + $scope.config.treeData = TreeDataService.getTreeData(); + }); + } - $scope.getChildrenFromDB = function(item){ - console.log("ALX:gcfd:" + JSON.stringify(item)); - return $scope.callPythonDo({ method: "get_children_from_db", parent: item }) - .then(function (data) { - console.log("ALX:data1=" + JSON.stringify(data)); - item.children = data.choices; - item.children.forEach(child => { - child.expanded = false; + }; + + $scope.updateDatas = function () { + $scope.cleanTree(); + $scope.initializeTree(); + $scope.getTemplatesFromDB(); + $scope.getCategoriesFromDB(); + $scope.showTreeData = false; + } + + $scope.getChildrenFromDB = function (item) { + console.log("ALX:gcfd:" + JSON.stringify(item)); + return $scope.callPythonDo({ method: "get_children_from_db", parent: item }) + .then(function (data) { + console.log("ALX:data1=" + JSON.stringify(data)); + item.children = data.choices; + item.children.forEach(child => { + child.expanded = false; + }); + console.log(item); + return item; + }); + } + + $scope.getTemplateHierarchyFromDB = function (item) { + console.log("ALX:gthfd:" + JSON.stringify(item)); + return $scope.callPythonDo({ method: "get_template_hierarchy_from_db", parent: item }) + .then(function (data) { + console.log("ALX:data1=" + JSON.stringify(data)); + item.children = data.choices; + item.children.forEach(child => { + child.expanded = false; + }); + console.log(item); + return item; }); - console.log(item); - return item; + } + + $scope.getTemplatesFromDB = function () { + $scope.callPythonDo({ method: "get_templates_from_db" }).then(function (data) { + $scope.config.templates = data.choices; + TreeDataService.setTemplateTreeData(data.choices); + $scope.config.templateTreeData = TreeDataService.getTemplateTreeData(); }); - } - - - // Toggle récursif des checkboxes - $scope.toggleChildren = function(node) { - console.log("ALX:tc:" + JSON.stringify(node)); - node.expanded = !node.expanded; - $scope.getChildrenFromDB(node); - if (node.children && node.children.length) { - node.children.forEach(function(child) { - child.expanded = !child.expanded; - $scope.getChildrenFromDB(child); + } + + $scope.setTab = function(tab) { + $scope.config.activeTab = tab; + }; + + $scope.getCategoriesFromDB = function () { + $scope.config.attribute_categories = []; + $scope.config.element_categories = []; + $scope.callPythonDo({ method: "get_attribute_categories_from_db" }).then(function (data) { + $scope.config.attribute_categories = data.choices; + }); + $scope.callPythonDo({ method: "get_element_categories_from_db" }).then(function (data) { + $scope.config.element_categories = data.choices; }); } - - }; - $scope.doSearch = function(element_name, attribute_name){ - $scope.callPythonDo({method: "do_search", element_name: element_name, attribute_name: attribute_name, root_tree: $scope.config.treeData}).then( - function(data){ + // Toggle récursif des checkboxes + $scope.toggleChildren = function (node) { + console.log("ALX:tc:" + JSON.stringify(node)); + node.expanded = !node.expanded; + $scope.getChildrenFromDB(node); + if (node.children && node.children.length) { + node.children.forEach(function (child) { + child.expanded = !child.expanded; + $scope.getChildrenFromDB(child); + }); + } + + }; + + $scope.doSearch = function (element_name, attribute_name) { + $scope.callPythonDo({ method: "do_search", element_name: element_name, attribute_name: attribute_name, root_tree: $scope.config.treeData }).then( + function (data) { TreeDataService.setTreeData(data.choices); $scope.config.treeData = TreeDataService.getTreeData(); $scope.config.attributeList = data.attributes; @@ -116,122 +222,188 @@ app.controller('AfExplorerFormCtrl', [ ); }; - $scope.updateAttributeToOutput = function (attribute) { - if (attribute.checked && $scope.config.selectedAttributes.includes(attribute)) { - $scope.config.selectedAttributes = $scope.config.selectedAttributes.filter(attr => attr.path !== attribute.path); - } - else { - console.log("Adding attribute to output:", attribute); - - if (!$scope.config || !$scope.config.attributeList || $scope.config.selectedAttributes.includes(attribute)) { - return; - } - const attrInConfig = $scope.config.attributeList.find(attr => attr.path === attribute.path); - - if (attrInConfig) { - $scope.config.selectedAttributes.push(attribute); - attrInConfig.checked = true; + $scope.toggleSelectAllAttributes = function () { + if ($scope.config.selectAllAttributes) { + $scope.config.selectedAttributes = [...$scope.config.attributeList]; + $scope.config.attributeList.forEach(attr => attr.checked = true); } else { - console.warn("Attribute not found in config:", attribute.path); + $scope.config.selectedAttributes = []; + $scope.config.attributeList.forEach(attr => attr.checked = false); } } - }; + $scope.updateAttributeToOutput = function (attribute) { + if (!$scope.config || !$scope.config.attributeList) return; -$scope.displayAttributes = function(node) { + const selectedAttributes = $scope.config.selectedAttributes; + const attributeList = $scope.config.attributeList; - if (!node.children || node.children.length === 0) { - $scope.getChildrenFromDB(node).then(newNode => { - processNode(newNode); - }); - } else { - processNode(node); - }; + const index = selectedAttributes.findIndex(attr => attr.path === attribute.path); + + if (index !== -1) { + selectedAttributes.splice(index, 1); + + const attrInConfig = attributeList.find(attr => attr.path === attribute.path); + if (attrInConfig) attrInConfig.checked = false; + + $scope.config.selectAllAttributes = false; + return; } -function processNode(node) { - $scope.config.attributeList = []; - $scope.config.selectedAttributes = []; - node.children.forEach(child => { - if (child.type === "attribute") { - $scope.config.attributeList.push({ - "name": child.title, - "path": child.path - }); - } - }); -} - - -}]); - -app.directive('treeNode', function() { - return { - restrict: 'E', - scope: { node: '=' }, - template: ` -
- - - ▼ - ▶ - + const attrInConfig = attributeList.find(attr => attr.path === attribute.path); - + if (!attrInConfig) { + console.warn("Attribute not found in config:", attribute.path); + return; + } - + selectedAttributes.push(attribute); + attrInConfig.checked = true; -
+ $scope.config.selectAllAttributes = selectedAttributes.length === attributeList.length; +}; - {{ node.title }} -
-
+ $scope.displayAttributes = function (node) { + $scope.config.selectAllAttributes = false; + if (!node.children || node.children.length === 0) { - - `, - link: function(scope) { - scope.toggleChildren = scope.$parent.toggleChildren; - scope.getChildrenFromDB = scope.$parent.getChildrenFromDB; - scope.doSearch = scope.$parent.doSearch; - scope.config = scope.$parent.config; - scope.attributeList = scope.config.attributeList || []; - scope.displayAttributes = scope.$parent.displayAttributes; - scope.toggleExpand = function(node) { - node.expanded = !node.expanded; - - if (node.expanded && (!node.children || !node.children.length)) { - scope.getChildrenFromDB(node); + if (node.type === "element") { + $scope.config.template = "-- Any --"; + $scope.getChildrenFromDB(node).then(newNode => { + processNode(newNode); + }); + } else if (node.type === "template") { + $scope.config.element_name = "*"; + $scope.config.template = node.title; + $scope.doSearch($scope.config.element_name, $scope.config.attribute_name); } + } else { + processNode(node); }; + } - scope.hasAttributes = function(node) { - if (!Array.isArray(scope.$parent.config.attributeList) || scope.$parent.config.attributeList.length === 0) { - return false; + function processNode(node) { + if (node.title !== $scope.config.element_name) { + $scope.config.element_name = ""; + } + $scope.config.attributeList = []; + $scope.config.selectedAttributes = []; + node.children.forEach(child => { + if (child.type === "attribute") { + $scope.config.attributeList.push(child); } + }); + } + + + }]); - return scope.$parent.config.attributeList.some(child => { - const expected = node.title + "|" + child.title; - return child.path.endsWith(expected); - }); - }; - scope.isElement = function(child) { - return child.type === 'element'; + +app.component('treeNode', { + bindings: { + node: '=', + getChildrenFromDb: '<', + displayAttributes: '<', + config: '<', + clickedNodes: '=' + }, + + controllerAs: 'ctrl', + + controller: function () { + const ctrl = this; + + ctrl.toggleExpand = function (node, $event) { + if ($event) { + $event.stopPropagation(); } - } - }; + + node.expanded = !node.expanded; + + if (node.expanded && (!node.children || !node.children.length)) { + // Call function reference directly + ctrl.getChildrenFromDb(node); + } + }; + + ctrl.onNodeClick = function (node) { + const index = ctrl.config.clickedNodes.indexOf(node.url); + ctrl.config.clickedNodes = []; // TODO remove when you want to use multiselect + ctrl.displayAttributes(node); + if (index > -1) { + ctrl.config.clickedNodes.splice(index, 1); + } else { + ctrl.config.clickedNodes.push(node.url); + } + }; + + ctrl.hasAttributes = function (node) { + if ( + !Array.isArray(ctrl.config?.attributeList) || + !ctrl.config.attributeList.length + ) { + return false; + } + + return ctrl.config.attributeList.some(attr => { + const expected = node.title + '|' + attr.title; + return attr.path.endsWith(expected); + }); + }; + + ctrl.isNodeClicked = function (node) { + return ctrl.config.clickedNodes.includes(node.url); + }; + }, + + template: ` +
+ + + ▼ + ▶ + + +
+ + {{ ctrl.node.title }} + +
+ +
+ + + ` }); + diff --git a/plugin.json b/plugin.json index 1b199f65..4494e851 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "id": "pi-system", - "version": "1.4.1", + "version": "1.4.2", "meta": { "label": "PI System", "description": "Retrieve data from your OSIsoft PI System servers", diff --git a/python-lib/osisoft_client.py b/python-lib/osisoft_client.py index 71c39c1e..8f9b083d 100644 --- a/python-lib/osisoft_client.py +++ b/python-lib/osisoft_client.py @@ -748,6 +748,122 @@ def search_attributes(self, database_webid, **kwargs): else: json_response = None + def search_elements(self, database_webid, name=None, description=None, category=None, template=None, full_search=True): + headers = self.get_requests_headers() + tempo_maxcount = OSIsoftConstants.DEFAULT_MAXCOUNT + params = { + "maxCount": tempo_maxcount, + "associations": "Paths", + } + url = self.endpoint.get_base_url() + "/assetdatabases/{}/elements".format(database_webid) + if name: + params["nameFilter"] = name + if description: + params["descriptionFilter"] = description + if category: + params["categoryName"] = category + if template: + params["templateName"] = template + if full_search: + params["searchFullHierarchy"] = True + json_response = self.get(url=url, headers=headers, params=params) + if OSIsoftConstants.DKU_ERROR_KEY in json_response: + yield json_response + start_index = 0 + while json_response: + items = json_response.get(OSIsoftConstants.API_ITEM_KEY, []) + for item in items: + yield item + if len(items) < tempo_maxcount: + logger.info("No more result items") + return + start_index += tempo_maxcount + logger.info("Trying again with startIndex={}".format(start_index)) + params["startIndex"] = start_index + json_response = self.get(url=url, headers=headers, params=params) + + def batched_search(self, database, element_name, attribute_name, element_category, + attribute_category, template, restrict_to_elements, + elements_max_count=None, attributes_max_count=None): + elements_query = { + "templateName": template, + "categoryName": element_category, + "nameFilter": element_name, + "searchFullHierarchy": "true", + "associations": "Paths" + } + if elements_max_count: + elements_query["maxCount"] = elements_max_count + attribute_query = { + "searchFullHierarchy": "true", + "associations": "Paths" + } + if attribute_name: + attribute_query["nameFilter"] = attribute_name + if attribute_category: + attribute_query["categoryName"] = attribute_category + if attributes_max_count: + attribute_query["maxCount"] = attributes_max_count + elements_url = "{}/elements".format(database) + if not restrict_to_elements: + request_body = { + "elements": { + "Method": "GET", + "Resource": "{}{}".format( + elements_url, + build_query_string("", elements_query) + ) + }, + "attributes": { + "Method": "GET", + "RequestTemplate": { + "Resource": "{{0}}{}".format( + build_query_string("", attribute_query) + ) + }, + "ParentIds": ["elements"], + "Parameters": ["$.elements.Content.Items[*].Links.Attributes"] + } + } + url = self.endpoint.get_batch_endpoint() + headers = OSIsoftConstants.WRITE_HEADERS + response = self.post(url, headers=headers, data=request_body, params={}) + json_response = response.json() + attributes = json_response.get("attributes", {}) + attributes_content = attributes.get("Content", {}) + if not isinstance(attributes_content, dict): + # the search returned nothing + return + attributes_content_items = attributes_content.get("Items", []) + for attributes_content_item in attributes_content_items: + content = attributes_content_item.get("Content", {}) + sub_items = content.get("Items", []) + for sub_item in sub_items: + yield sub_item + else: + count = 1 + request_body = {} + for restrict_to_element in restrict_to_elements: + job_tag = "J_{}".format(count) + request_body[job_tag] = { + "Method": "GET", + "Resource": "{}/attributes{}".format( + restrict_to_element, + build_query_string("", attribute_query) + ) + } + count = 1 + url = self.endpoint.get_batch_endpoint() + headers = OSIsoftConstants.WRITE_HEADERS + response = self.post(url, headers=headers, data=request_body, params={}) + json_response = response.json() + for job_tag in json_response: + job_result = json_response.get(job_tag) + content = job_result.get("Content", {}) + sub_items = content.get("Items", []) + for sub_item in sub_items: + yield sub_item + def build_element_query(self, **kwargs): element_query_keys = { "element_name": "Name:'{}'", @@ -857,7 +973,7 @@ def traverse_and_cache(self, path_elements, path_attributes, tree): for path_attribute in path_attributes: item = self.extract_item_with_name(json_response, path_attribute) item_details = get_item_details(item) - item_details["checked"] = True # That should not be done here + # item_details["checked"] = True # That should not be done here tree.put(full_path_elements[0:counter], item_details) counter += 1 next_url = self.extract_link_with_key(item, "Attributes") @@ -866,6 +982,8 @@ def traverse_and_cache(self, path_elements, path_attributes, tree): json_response = self.get(url=next_url, headers=headers, params={}, error_source="traverse_and_cache") else: break + if not before_last_json: + return None items = before_last_json.get(OSIsoftConstants.API_ITEM_KEY, []) for item in items: item_details = get_item_details(item) @@ -1091,7 +1209,7 @@ def close(self): def validate_timestamp(timestamp): - valid_formats=["%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"] + valid_formats = ["%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"] for valid_format in valid_formats: try: datetime.strptime(timestamp, valid_format) @@ -1115,7 +1233,7 @@ def build_query_string(url, params): if isinstance(value, list): for element in value: tokens.append(key+"="+str(element)) - else: + elif value is not None: tokens.append(key+"="+str(value)) if len(tokens) > 0: return url + "?" + "&".join(tokens) diff --git a/python-lib/osisoft_constants.py b/python-lib/osisoft_constants.py index ce9b74ab..ff6854b4 100644 --- a/python-lib/osisoft_constants.py +++ b/python-lib/osisoft_constants.py @@ -405,7 +405,7 @@ class OSIsoftConstants(object): "Security": "{base_url}/eventframes/{webid}/security", "SecurityEntries": "{base_url}/eventframes/{webid}/securityentries" } - PLUGIN_VERSION = "1.4.1-beta.1" + PLUGIN_VERSION = "1.4.2-beta.1" VALUE_COLUMN_SUFFIX = "_val" WEB_API_PATH = "piwebapi" WRITE_HEADERS = {'X-Requested-With': 'XmlHttpRequest'} diff --git a/python-lib/osisoft_plugin_common.py b/python-lib/osisoft_plugin_common.py index 82cc5802..4728423f 100644 --- a/python-lib/osisoft_plugin_common.py +++ b/python-lib/osisoft_plugin_common.py @@ -154,6 +154,7 @@ def build_requests_params(**kwargs): "boundary_type": "syncTimeBoundaryType", "name_filter": "nameFilter", "category_name": "categoryName", + "description": "descriptionFilter", "template_name": "templateName", "referenced_element_name_filter": "referencedElementNameFilter", "referenced_element_template": "referencedElementTemplate", @@ -638,8 +639,8 @@ def add(self, start_time, end_time, interval): def get_item_details(item): KEYS_TO_CHECK = { - "Name": "title", "TemplateName": "template_name", "CategoryNames": "category_names", - "HasChildren": "has_children", "Path": "path", "WebId": "id", "checked": "checked" + "Name": "title", "TemplateName": "template_name", "CategoryNames": "category_names", "Description": "description", + "HasChildren": "has_children", "Path": "path", "WebId": "id", "checked": "checked", "BaseTemplate": "BaseTemplate" } # should we stick to python naming convention or keep pi's ones throughout ? details = {} for key_to_check in KEYS_TO_CHECK: diff --git a/resource/browse_af_tree.py b/resource/browse_af_tree.py index 91b3f6dd..5a4dca05 100644 --- a/resource/browse_af_tree.py +++ b/resource/browse_af_tree.py @@ -1,20 +1,13 @@ from osisoft_client import OSIsoftClient +from safe_logger import SafeLogger from osisoft_plugin_common import get_credentials, build_select_choices, check_debug_mode -from osisoft_plugin_common import get_item_details, Tree, recursive_tree_rebuild +from osisoft_plugin_common import get_item_details, Tree, recursive_tree_rebuild, PerformanceTimer import dataiku +logger = SafeLogger("PI System plugin", ["user", "password"]) + def do(payload, config, plugin_config, inputs): - input_tree = None - if len(inputs) > 0: - input_item = inputs[0] - input_type = input_item.get("type") - if input_type == "DATASET": - input_dataset_name = input_item.get("fullName") - input_dataset = dataiku.Dataset(input_dataset_name) - input_tree = input_dataset.get_dataframe(infer_with_pandas=False) - - config["is_ssl_check_disabled"] = True if "config" in config: config = config.get("config") if "credentials" not in config: @@ -23,6 +16,7 @@ def do(payload, config, plugin_config, inputs): return {"choices": [{"label": "Pick a credential"}]} auth_type, username, password, server_url, is_ssl_check_disabled, credential_error = get_credentials(config, can_raise=False) + is_ssl_check_disabled = config.get("is_ssl_check_disabled", False) # Because no advanced parameter switch if credential_error: return build_select_choices(credential_error) @@ -40,9 +34,14 @@ def do(payload, config, plugin_config, inputs): return build_select_choices("Fill in the server address") is_debug_mode = check_debug_mode(config) - is_ssl_check_disabled = True - client = OSIsoftClient(server_url, auth_type, username, password, is_ssl_check_disabled=is_ssl_check_disabled, is_debug_mode=is_debug_mode) + network_timer = PerformanceTimer() + + client = OSIsoftClient( + server_url, auth_type, username, password, + is_ssl_check_disabled=is_ssl_check_disabled, is_debug_mode=is_debug_mode, + network_timer=network_timer + ) method = payload.get("method") if method == "get_query_catalogs": @@ -51,7 +50,36 @@ def do(payload, config, plugin_config, inputs): database_name = config.get("database_name") parent = payload.get("parent", {}) return get_children_from_db(client, parent, database_name=database_name) + if method == "get_templates_from_db": + database_name = config.get("database_name") + parent = payload.get("parent", {}) + # ret = get_items_from_db(client, parent, "ElementTemplates", database_name=database_name) + ret = get_template_hierarchy_from_db(client, parent, database_name=database_name) + return ret + if method == "get_attribute_categories_from_db": + database_name = config.get("database_name") + parent = payload.get("parent", {}) + ret = get_items_from_db(client, parent, "AttributeCategories", database_name=database_name) + return ret + if method == "get_element_categories_from_db": + database_name = config.get("database_name") + parent = payload.get("parent", {}) + ret = get_items_from_db(client, parent, "ElementCategories", database_name=database_name) + return ret if method == "do_search": + template_name = config.get("template", None) + category_name = config.get("element_category", None) + clicked_nodes = config.get("clickedNodes", []) + if template_name == "-- Any --": + template_name = None + if category_name == "-- Any --": + category_name = None + element_category = config.get("element_category", None) + if element_category == "-- Any --": + element_category = None + attribute_category = config.get("attribute_category", None) + if attribute_category == "-- Any --": + attribute_category = None database_name = config.get("database_name") element_name = config.get("element_name") attribute_name = config.get("attribute_name") @@ -61,26 +89,15 @@ def do(payload, config, plugin_config, inputs): attributes = [] # https://dku-qa-osi.francecentral.cloudapp.azure.com/piwebapi/assetdatabases/F1RD3VEt1yTvt0ip6-a5yeEVsgbMcrwu_Je0qg9btcZIvPswT1NJU09GVC1QSS1TRVJWXFdFTEw database_webid = database_name.split("/")[-1] - # element_query_keys = { - # "element_name": "Name:'{}'", - # "search_root_path": "Root:'{}'", - # "element_template": "Template:'{}'", - # "element_type": "Type:'{}'", - # "element_category": "CategoryName:'{}'" - # } - # attribute_query_keys = { - # "attribute_name": "Name:'{}'", - # "attribute_category": "CategoryName:'{}'", - # "attribute_value_type": "Type:'{}'" - # } - for attribute in client.search_attributes( - database_webid, - attribute_name=attribute_name, - element_name=element_name, - search_associations="Paths" - ): - attribute["checked"] = False - attributes.append(attribute) + elements_max_count, attributes_max_count = get_max_counts(config) + + attributes = [] + for result in client.batched_search(database_name, element_name, attribute_name, + element_category, attribute_category, template_name, clicked_nodes, + elements_max_count=elements_max_count, attributes_max_count=attributes_max_count): + # result["checked"] = True + attributes.append(result) + attributes = duplicate_linked_attributes(attributes) items = [] for attribute in attributes: @@ -88,6 +105,7 @@ def do(payload, config, plugin_config, inputs): items.append(item) attributesCopy = items.copy() rebuilt_tree = rebuild_tree(client, items, root_tree) + logger.info("Search network timer:{}".format(network_timer.get_report())) return {"choices": rebuilt_tree, "attributes": attributesCopy} parameter_name = payload.get("parameterName") @@ -123,6 +141,25 @@ def get_query_catalogs(cnx, config): return {"choices": [user, password]} +def get_items_from_db(client, parent_node, link_key, database_name=None): + default_choice = {"title": "-- Any --"} + if isinstance(parent_node, dict): + url = parent_node.get("url", database_name) + else: + url = parent_node + this_node = next(client.get_next_item_from_url(url)) + links = this_node.get("Links", {}) + items_url = links.get(link_key) + items = [] + items.append(default_choice) + if items_url: + for item in client.get_next_item_from_url(items_url): + item = get_item_details(item) + item["type"] = link_key + items.append(item) + return {"choices": items} + + def get_children_from_db(client, parent_node, database_name=None): if isinstance(parent_node, dict): url = parent_node.get("url", database_name) @@ -150,19 +187,51 @@ def get_children_from_db(client, parent_node, database_name=None): if child.get("has_children"): child["children"] = [] children.append(child) - return {"choices": children} -# method2: -# we dig, but this time it's index[token name], and we store as we go in the child, with the real data indexed in a list and just the rank pointing to it -# to build the final tree, we browse the index, get the index data, rebuild the struct from there -# Tree class ? put(path, data), get(path, data) + +def get_template_hierarchy_from_db(client, parent_node, database_name=None): + if isinstance(parent_node, dict): + url = parent_node.get("url", database_name) + else: + url = parent_node + default_choice = {"title": "-- Any --", "id:": ""} + this_node = next(client.get_next_item_from_url(url)) + links = this_node.get("Links", {}) + element_templates_url = links.get("ElementTemplates") + children = [default_choice] + rebuilt_tree = [] + if element_templates_url: + element_templates = client.get_next_item_from_url(element_templates_url) + for element_template in element_templates: + child = get_item_details(element_template) + child["type"] = "template" + child["children"] = [] + children.append(child) + rebuilt_tree = nest_children(children) + return {"choices": rebuilt_tree} + + +def nest_children(items): + name_to_item = {item["title"]: item for item in items} + tree = [] + for item in items: + parent_name = item.get("BaseTemplate") + if parent_name is None or parent_name not in name_to_item: + tree.append(item) + else: + parent = name_to_item[parent_name] + if "children" not in parent: + parent["children"] = [] + parent["children"].append(item) + parent["has_children"] = True + return tree def rebuild_tree(client, items, root_tree=None): # builds an active tree containing all the items and their parent up to the root tree = Tree(root_tree=root_tree) - tree.print() + # tree.print() while len(items) > 1: item = items.pop() if item is None: @@ -239,3 +308,24 @@ def set_as_selected(items): def update_item(item, tree): elements_paths_tokens, attributes_paths_tokens = path_to_list(item.get("path")) tree.put(elements_paths_tokens + attributes_paths_tokens, item) + + +def get_max_counts(config): + show_advanced_parameters = config.get("show_advanced_parameters", False) + if not show_advanced_parameters: + return 100, 100 + + def parse_max_count(value, default): + if value is None or value == "": + return default + try: + value = int(value) + except (TypeError, ValueError): + return default + if value <= 0: + return None + return value + + elements_max_count = parse_max_count(config.get("elements_max_count"), 100) + attributes_max_count = parse_max_count(config.get("attributes_max_count"), 100) + return elements_max_count, attributes_max_count diff --git a/resource/pi-system_af-explorer.css b/resource/pi-system_af-explorer.css index 575492b3..94d138da 100644 --- a/resource/pi-system_af-explorer.css +++ b/resource/pi-system_af-explorer.css @@ -14,3 +14,78 @@ .pi-system-explorer__tree-view, .pi-system-explorer__center-view { border: 1px solid #ccc; padding: 10px; border-radius: 5px; } + +.pi-system-explorer__authentication-header { + background-color: #f0f0f0; + padding: 10px; + cursor: pointer; + border: 1px solid #ccc; + border-radius: 5px; + font-weight: bold; +} + +.pi-system-explorer__authentication-body { + border: 1px solid #ccc; padding: 10px; border-radius: 5px; +} + +.pi-system-config--mandatory { + display: flex; + gap: 32px; +} + +.custom-table { + border-collapse: collapse; + width: 100%; + font-family: Arial, sans-serif; +} + +.custom-table th, +.custom-table td { + border: 1px solid #ccc; + padding: 8px 12px; + text-align: left; +} + +.custom-table th { + background-color: #f4f4f4; + font-weight: bold; +} + +.custom-table tr:nth-child(even) { + background-color: #fafafa; +} + +.custom-table tr:hover { + background-color: #f1f7ff; +} + +.tab-container { + font-family: Arial, sans-serif; +} + +.tab-header { + list-style: none; + padding: 0; + margin: 0 0 6px 0; + display: flex; + border-bottom: 2px solid #ddd; +} + +.tab-header li { + padding: 8px 16px; + cursor: pointer; + background: #f5f5f5; + border: 1px solid #ddd; + border-bottom: none; + margin-right: 4px; +} + +.tab-header li.active { + background: #fff; + font-weight: bold; +} + +.tab-content { + border-top: none; + padding: 10px; +} diff --git a/resource/pi-system_af-explorer.html b/resource/pi-system_af-explorer.html index d331c889..942a51f7 100644 --- a/resource/pi-system_af-explorer.html +++ b/resource/pi-system_af-explorer.html @@ -1,93 +1,167 @@
-
- -
-
+
+
+ Welcome to Pi System Plugin +
+
+
+
+ +
+
+
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+ +
+ +
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+ +
+ +
+
+
+ +
-
- -
- -
-
-
- -
- -
- -
-
- -
- -
- -
-
-
-
Elements
-
- - -
-
    -
  • - +
    + +
    + +
    +
    +
    +
    +
    +
      +
    • + Element +
      • +
    • + Template
    -
    - - -
    -
    Attributes
    -
    - -
    - + + +
    + +
    + + + +
    + +
    +
    + + + + + + + + + - + -
    TitleDescription
    - + {{attribute.title}}{{attribute.path}}{{attribute.description}}
    -
    - + +
    - -
    - +
\ No newline at end of file