From 4311ea6ffd4c24d000bd36d76102f0be4c395464 Mon Sep 17 00:00:00 2001 From: Thiago Date: Thu, 20 Mar 2025 06:35:56 -0300 Subject: [PATCH 1/5] Update load_api.py with account validation and new update endpoint; add old app files to .gitignore --- .gitignore | 32 +++++++++++ app/services/load_api.py | 111 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 139 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index d5b476f..610ab68 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,35 @@ cython_debug/ #.idea/ config/config.ini config/deployed_contracts.ini +app_old/api_server.py +app_old/config.ini +app_old/database_methods.py +app_old/deployed_contracts.ini +app_old/import_csv.py +app_old/instance_app.py +app_old/model.py +app_old/serealize.py +app_old/api/__init__.py +app_old/api/core_api.py +app_old/api/query_api.py +app_old/api/user_api.py +app_old/api/util.py +app_old/config/BlockchainManager.py +app_old/database/__init__.py +app_old/database/users.csv +app_old/database/variables_database.py +app_old/migrations/alembic.ini +app_old/migrations/env.py +app_old/migrations/README +app_old/migrations/script.py.mako +app_old/migrations/versions/2da74c140ea0_.py +app_old/static/dark-logo.png +app_old/templates/base.html +app_old/templates/home.html +app_old/templates/wallet.html +app_old/util/asynchronus.py +app_old/util/auth_middleware.py +app_old/util/config_manager.py +app_old/util/responses.py +app_old/util/synchronous.py +app_old/util/validation.py diff --git a/app/services/load_api.py b/app/services/load_api.py index 20d94a4..4d1764a 100644 --- a/app/services/load_api.py +++ b/app/services/load_api.py @@ -2,6 +2,7 @@ import os import sys import time +from datetime import datetime import random import configparser @@ -42,6 +43,13 @@ def get_bc_gateway(account_pk): return dark_gw, dark_map +def check_account(data): + dnam_pk = data.get('dnam_pk').strip() + account = Account.from_key(dnam_pk) + dnam_wallet = account.address + dark_gw, dark_map = get_bc_gateway(dnam_pk) + return account, dnam_wallet, dark_gw, dark_map + @load_api.route('/load', methods=['post']) def check_json(): data = request.json @@ -52,10 +60,11 @@ def check_json(): r_time = 0 try: - dnam_pk = data.get('dnam_pk').strip() - account = Account.from_key(dnam_pk) - dnam_wallet = account.address - dark_gw, dark_map = get_bc_gateway(dnam_pk) + account, dnam_wallet, dark_gw, dark_map = check_account(data) + # dnam_pk = data.get('dnam_pk').strip() + # account = Account.from_key(dnam_pk) + # dnam_wallet = account.address + # dark_gw, dark_map = get_bc_gateway(dnam_pk) except AttributeError: # sem a chave erros.append("No private key identified") @@ -144,3 +153,97 @@ def check_json(): return jsonify(response), 200 + +@load_api.route('/update', methods=['post']) +def check_json(): + data = request.json + items = data.get('items', []) + erros = [] + dnam_pk = '' + load_time = 0 + r_time = 0 + + try: + account, dnam_wallet, dark_gw, dark_map = check_account(data) + except AttributeError: + # sem a chave + erros.append("No private key identified") + except Exception: + erros.append("Invalid private key") + except ValueError: + erros.append("Invalid private key") + + # recuperar o pid + try: + dark_id = data.get('dark_id') + if not dark_id: + erros.append('No dark_id provided') + except ValueError as e: + erros.append('No dark_id provided') + try: + url = data.get('url') + if not dark_id: + erros.append('No url provided') + except ValueError as e: + erros.append('No url provided') + + + try: + dark_pid = dark_map.get_pid_by_ark(dark_id) + except Exception as e: + erros.append('dark_id not founded') + # resp = jsonify({'status' : 'Unable to recovery (' + str(dark_id) + ')', 'block_chain_error' : str(e)},) + # resp_code = 500 # colocar um erro especifico? + + if len(erros) > 0: + resp = {'erros' : erros , 'params' : str(data)} + return jsonify(resp), 500 + + item_data = {'dark_id': dark_id , 'old_url': dark_pid.external_url, 'update_url' : url} + if is_valid_url(url): + try: + ark_hash_obj = dark_pid.pid_hash + ark_hash_hex_val = ark_hash_obj.hex() + item_data['ark_hash'] = ark_hash_hex_val + tx = dark_map.async_set_url(ark_hash_obj,url) + txr = tx.hex() + item_data['tx_recipt'] = txr + except Exception as e: + item_data['error'] = 'bockchain error' + item_data['error_desc'] = str(e) + else: + item_data['error'] = 'invalid url' + + + + # recuperando os arks + # start_time = time.time() + # saida = [] + # for park in processados: + # ark_hash = park['ark_hash'] + # ark = dark_map.get_pid_by_hash(ark_hash).to_dict() + # park['ark'] = ark['ark'] + # park['ark_url'] = ark['external_url'] + + # saida.append(park) + # end_time = time.time() + # r_time = end_time - start_time + + + response = { + "wallet_addr": account.address , + "action": 'update', + "data" : item_data, + "timestamp": datetime.now().timestamp() + } + + # + os.makedirs('logs', exist_ok=True) + timestamp = int(time.time()) + random_digits = random.randint(000, 999) + filename = f"logs/update_{time.strftime('%Y-%m-%d')}_{dnam_wallet[2:]}_{timestamp}_{random_digits}.json" + with open(filename, 'w') as f: + json.dump(response, f, indent=4) + + return jsonify(response), 200 + From 0ad64ccceb417566fe4267aa5bcc8f6bd92615cb Mon Sep 17 00:00:00 2001 From: Thiago Date: Sat, 29 Mar 2025 11:42:40 -0300 Subject: [PATCH 2/5] Refactor update endpoint to improve error handling and processing logic; add test cases for various UPDATE scenarios --- app/services/load_api.py | 130 ++++++++++++++++++++++----------------- tests/update.http | 98 +++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 56 deletions(-) create mode 100644 tests/update.http diff --git a/app/services/load_api.py b/app/services/load_api.py index 4d1764a..8dadbe1 100644 --- a/app/services/load_api.py +++ b/app/services/load_api.py @@ -155,13 +155,11 @@ def check_json(): @load_api.route('/update', methods=['post']) -def check_json(): +def update_metadata(): data = request.json items = data.get('items', []) erros = [] - dnam_pk = '' - load_time = 0 - r_time = 0 + update_time = 0 try: account, dnam_wallet, dark_gw, dark_map = check_account(data) @@ -172,71 +170,91 @@ def check_json(): erros.append("Invalid private key") except ValueError: erros.append("Invalid private key") - - # recuperar o pid - try: - dark_id = data.get('dark_id') - if not dark_id: - erros.append('No dark_id provided') - except ValueError as e: - erros.append('No dark_id provided') - try: - url = data.get('url') - if not dark_id: - erros.append('No url provided') - except ValueError as e: - erros.append('No url provided') - - - try: - dark_pid = dark_map.get_pid_by_ark(dark_id) - except Exception as e: - erros.append('dark_id not founded') - # resp = jsonify({'status' : 'Unable to recovery (' + str(dark_id) + ')', 'block_chain_error' : str(e)},) - # resp_code = 500 # colocar um erro especifico? - + + # caso tenha parametros invalidos if len(erros) > 0: resp = {'erros' : erros , 'params' : str(data)} return jsonify(resp), 500 + + processados = [] + nao_processados = [] + start_time = time.time() + for item in items: - item_data = {'dark_id': dark_id , 'old_url': dark_pid.external_url, 'update_url' : url} - if is_valid_url(url): + # recuperar o pid + parametros_validos = True try: - ark_hash_obj = dark_pid.pid_hash - ark_hash_hex_val = ark_hash_obj.hex() - item_data['ark_hash'] = ark_hash_hex_val - tx = dark_map.async_set_url(ark_hash_obj,url) - txr = tx.hex() - item_data['tx_recipt'] = txr - except Exception as e: - item_data['error'] = 'bockchain error' - item_data['error_desc'] = str(e) - else: - item_data['error'] = 'invalid url' - - - - # recuperando os arks - # start_time = time.time() - # saida = [] - # for park in processados: - # ark_hash = park['ark_hash'] - # ark = dark_map.get_pid_by_hash(ark_hash).to_dict() - # park['ark'] = ark['ark'] - # park['ark_url'] = ark['external_url'] + dark_id = item.get('dark_id') + if not dark_id: + item['error'] = 'No valid dark_id provided' + nao_processados.append(item) + parametros_validos = False + except ValueError as e: + erros.append('No dark_id provided') + item['error'] = 'No valid dark_id provided' + nao_processados.append(item) + parametros_validos = False + try: + update_url = item.get('url') + item['update_url'] = update_url + del item['url'] + + if not update_url: + item['error'] = 'No url provided' + nao_processados.append(item) + parametros_validos = False + except ValueError as e: + item['error'] = 'No url provided' + nao_processados.append(item) + parametros_validos = False - # saida.append(park) - # end_time = time.time() - # r_time = end_time - start_time + if parametros_validos: + try: + dark_pid = dark_map.get_pid_by_ark(dark_id) + except Exception as e: + item['error'] = 'dark_id not found' + nao_processados.append(item) + break + # resp = jsonify({'status' : 'Unable to recovery (' + str(dark_id) + ')', 'block_chain_error' : str(e)},) + # resp_code = 500 # colocar um erro especifico? + + item['update_url'] = update_url + + if is_valid_url(update_url): + try: + ark_hash_obj = dark_pid.pid_hash + ark_hash_hex_val = ark_hash_obj.hex() + item['ark_hash'] = ark_hash_hex_val + tx = dark_map.async_set_url(ark_hash_obj,update_url) + txr = tx.hex() + item['tx_recipt'] = txr + item['previous_url'] = dark_pid.external_url + processados.append(item) + except Exception as e: + item['error'] = 'bockchain error' + item['error_desc'] = str(e) + nao_processados.append(item) + else: + item['error'] = 'invalid url' + nao_processados.append(item) + print("aqui") + + end_time = time.time() + update_time = end_time - start_time response = { "wallet_addr": account.address , "action": 'update', - "data" : item_data, - "timestamp": datetime.now().timestamp() + "update_time" : update_time, + "timestamp": datetime.now().timestamp(), + "update_pids": processados } + if len(nao_processados) != 0: + response["not_updated_pids"] = nao_processados + + # os.makedirs('logs', exist_ok=True) timestamp = int(time.time()) diff --git a/tests/update.http b/tests/update.http new file mode 100644 index 0000000..18caeca --- /dev/null +++ b/tests/update.http @@ -0,0 +1,98 @@ +### Variables +@apiurl = http://127.0.0.1:7000 + +# note : rmber to change de ark for valid arks + + +### load invalido 1 +POST {{apiurl}}/update HTTP/1.1 +Content-Type: application/json + +{ + "dnam_pk": "R1", + "items": [ + {"oai_id" : "a" , "url" : "invalida"}, + {"oai_id" : "b" , "url" : "http://uol.com"}, + {"oai_id" : "c" , "url" : "http://uol.com"}, + {"oai_id" : "d" , "url" : "http://uol.com"} + ] + +} + +### load invalido 2 +POST {{apiurl}}/update HTTP/1.1 +Content-Type: application/json + +{ + "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f", + "items": [ + {"oai_id" : "a" , "url" : "invalida"} + ] + +} + +### load invalido 3 +POST {{apiurl}}/update HTTP/1.1 +Content-Type: application/json + +{ + "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f", + "items": [ + {"oai_id" : "a" , "url" : "invalida"}, + {"dark_id" : "b" , "url" : "http://uol.com"} + ] + +} + +### load invalida 4 +POST {{apiurl}}/update HTTP/1.1 +Content-Type: application/json + +{ + "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f", + "items": [ + {"dark_id" : "8033/fkwf300000004h" , "url" : "http:/gov.com/"} + ] + +} + +### load valida 1 +POST {{apiurl}}/update HTTP/1.1 +Content-Type: application/json + +{ + "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f", + "items": [ + {"dark_id" : "8033/fkwf30000003gg" , "url" : "http://uol.com/up1"} + ] + +} + + +### load valida 2 +POST {{apiurl}}/update HTTP/1.1 +Content-Type: application/json + +{ + "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f", + "items": [ + {"dark_id" : "8033/fkwf300000001n" , "url" : "http://nova.com/up1"}, + {"dark_id" : "8033/fkwf3000000028" , "url" : "http://new.com/up1"}, + {"dark_id" : "8033/fkwf300000004h" , "url" : "http://nueva.com/up1"} + ] + +} + +### load invalida 4 +POST {{apiurl}}/update HTTP/1.1 +Content-Type: application/json + +{ + "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f", + "items": [ + {"dark_id" : "8033/fkwf300000004h" , "url" : "http:/gov.com/"}, + {"dark_id" : "8033/fkwf3000000028" , "url" : "http://tudook.com/update"} + ] + +} + From 49eefe25359f441631658a6170f922320044829c Mon Sep 17 00:00:00 2001 From: Thiago Date: Sat, 29 Mar 2025 11:56:18 -0300 Subject: [PATCH 3/5] Rename key in update_metadata function from 'update_pids' to 'updated_pids' for clarity --- app/services/load_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/load_api.py b/app/services/load_api.py index 8dadbe1..f03223a 100644 --- a/app/services/load_api.py +++ b/app/services/load_api.py @@ -248,7 +248,7 @@ def update_metadata(): "action": 'update', "update_time" : update_time, "timestamp": datetime.now().timestamp(), - "update_pids": processados + "updated_pids": processados } if len(nao_processados) != 0: From d50ced9683c7efa5c88bc1317851776ae1fbe5b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thiago=20N=C3=B3brega?= Date: Tue, 22 Apr 2025 18:19:35 -0300 Subject: [PATCH 4/5] Add url_exists utility function and improve error handling in load_data function --- app/services/load_api.py | 28 +++++++++++++++++++++++----- app/util/bc.py | 7 +++++++ tests/README.md | 1 + 3 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 app/util/bc.py create mode 100644 tests/README.md diff --git a/app/services/load_api.py b/app/services/load_api.py index f03223a..096a4e2 100644 --- a/app/services/load_api.py +++ b/app/services/load_api.py @@ -12,6 +12,7 @@ from dark import DarkMap, DarkGateway from util.validation import is_valid_url +from util.bc import url_exists from eth_account import Account import shared_utils @@ -51,7 +52,7 @@ def check_account(data): return account, dnam_wallet, dark_gw, dark_map @load_api.route('/load', methods=['post']) -def check_json(): +def load_data(): data = request.json items = data.get('items', []) erros = [] @@ -69,10 +70,13 @@ def check_json(): # sem a chave erros.append("No private key identified") except Exception: - erros.append("Invalid private key") + erros.append("Invalid private key.") except ValueError: erros.append("Invalid private key") + if len(erros) > 0: + resp = {'erros' : erros , 'params' : str(data)} + return jsonify(resp), 500 processados = [] @@ -84,7 +88,8 @@ def check_json(): item_data = {'oai_id' : oai_id , 'requested_url' : url} - if oai_id and is_valid_url(url): + + if oai_id and ( is_valid_url(url) and url_exists(url,dark_map) == False ): try: ark_hash_obj = shared_utils.get_pid(dnam_wallet,dark_map) ark_hash_hex_val = ark_hash_obj.hex() @@ -100,7 +105,10 @@ def check_json(): item_data['error_desc'] = str(e) nao_processados.append(item_data) else: - item_data['error'] = 'invalid url' + if not is_valid_url(url): + item_data['error'] = 'invalid url' + else: + item_data['error'] = 'URL already exists and set to other ark' nao_processados.append(item_data) end_time = time.time() @@ -203,6 +211,16 @@ def update_metadata(): item['error'] = 'No url provided' nao_processados.append(item) parametros_validos = False + if not is_valid_url(update_url): + item['error'] = 'invalid URL' + nao_processados.append(item) + parametros_validos = False + + if url_exists(update_url,dark_map): + item['error'] = 'URL already exists and set to other ark' + nao_processados.append(item) + parametros_validos = False + except ValueError as e: item['error'] = 'No url provided' nao_processados.append(item) @@ -237,7 +255,7 @@ def update_metadata(): else: item['error'] = 'invalid url' nao_processados.append(item) - print("aqui") + # print("aqui") end_time = time.time() diff --git a/app/util/bc.py b/app/util/bc.py new file mode 100644 index 0000000..ac7f42d --- /dev/null +++ b/app/util/bc.py @@ -0,0 +1,7 @@ +from dark import DarkMap +# from utils import is_valid_url + +def url_exists(url: str, dm:DarkMap) -> bool: + # print(url, dm.url_db.caller.exist(url)) + return dm.url_db.caller.exist(url) + \ No newline at end of file diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..afe23a3 --- /dev/null +++ b/tests/README.md @@ -0,0 +1 @@ +Use Ress Client (github)[https://github.com/Huachao/vscode-restclient], (market)[https://marketplace.visualstudio.com/items/?itemName=humao.rest-client] to test \ No newline at end of file From afa837fed9c5d6dccf9c2904e6a334dcdbf8add9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thiago=20N=C3=B3brega?= Date: Wed, 23 Apr 2025 15:33:05 -0300 Subject: [PATCH 5/5] Enhance update function to check for existing URLs before processing updates --- README.md | 8 +++++--- app/services/load_api.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9d2f9e6..6c57d77 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ Follow these simple steps to run the project: ### System Requirements -- Python 3.10 +- Python 3.10 or higher - pip - Docker - Docker Compose @@ -82,10 +82,12 @@ Follow these simple steps to run the project: **linux** > ``` +> $ sudo apt install build-essential +> $ sudo apt install python3.12-dev > $ cd hyperdrive > $ pip install -r requirements > $ cd app -> $ python api_server.py +> $ python server.py > ``` **windows** @@ -93,7 +95,7 @@ Follow these simple steps to run the project: > cd hyperdrive > pip install -r requirements > cd app -> python.exe api_server.py +> python.exe server.py >``` ### Docker execution diff --git a/app/services/load_api.py b/app/services/load_api.py index 096a4e2..50b9c3e 100644 --- a/app/services/load_api.py +++ b/app/services/load_api.py @@ -216,6 +216,7 @@ def update_metadata(): nao_processados.append(item) parametros_validos = False + # chek whether the url already exists if url_exists(update_url,dark_map): item['error'] = 'URL already exists and set to other ark' nao_processados.append(item)