Update in dark 1

.gitignore

@@ -163,3 +163,35 @@ cython_debug/
 #.idea/
 config/config.ini
 config/deployed_contracts.ini
+app_old/api_server.py
+app_old/config.ini
+app_old/database_methods.py
+app_old/deployed_contracts.ini
+app_old/import_csv.py
+app_old/instance_app.py
+app_old/model.py
+app_old/serealize.py
+app_old/api/__init__.py
+app_old/api/core_api.py
+app_old/api/query_api.py
+app_old/api/user_api.py
+app_old/api/util.py
+app_old/config/BlockchainManager.py
+app_old/database/__init__.py
+app_old/database/users.csv
+app_old/database/variables_database.py
+app_old/migrations/alembic.ini
+app_old/migrations/env.py
+app_old/migrations/README
+app_old/migrations/script.py.mako
+app_old/migrations/versions/2da74c140ea0_.py
+app_old/static/dark-logo.png
+app_old/templates/base.html
+app_old/templates/home.html
+app_old/templates/wallet.html
+app_old/util/asynchronus.py
+app_old/util/auth_middleware.py
+app_old/util/config_manager.py
+app_old/util/responses.py
+app_old/util/synchronous.py
+app_old/util/validation.py

README.md

@@ -72,7 +72,7 @@ Follow these simple steps to run the project:
 
 ### System Requirements
 
-- Python 3.10
+- Python 3.10 or higher
 - pip
 - Docker
 - Docker Compose
@@ -82,18 +82,20 @@ Follow these simple steps to run the project:
 
 **linux**
 > ```
+> $ sudo apt install build-essential
+> $ sudo apt install python3.12-dev
 > $ cd hyperdrive
 > $ pip install -r requirements
 > $ cd app
-> $ python api_server.py
+> $ python server.py
 > ```
 
 **windows**
 > ```
 > cd hyperdrive
 > pip install -r requirements
 > cd app
-> python.exe api_server.py
+> python.exe server.py
 >```
 
 ### Docker execution

app/services/load_api.py

@@ -2,6 +2,7 @@
 import os
 import sys
 import time
+from datetime import datetime
 import random
 
 import configparser
@@ -11,6 +12,7 @@
 from dark import DarkMap, DarkGateway
 
 from util.validation import is_valid_url
+from util.bc import url_exists
 from eth_account import Account
 
 import shared_utils 
@@ -42,8 +44,15 @@ def get_bc_gateway(account_pk):
     return dark_gw, dark_map
 
 
+def check_account(data):
+    dnam_pk = data.get('dnam_pk').strip()
+    account = Account.from_key(dnam_pk)
+    dnam_wallet = account.address
+    dark_gw, dark_map = get_bc_gateway(dnam_pk)
+    return account, dnam_wallet, dark_gw, dark_map
+
 @load_api.route('/load', methods=['post'])
-def check_json():
+def load_data():
     data = request.json
     items = data.get('items', [])
     erros = []
@@ -52,18 +61,22 @@ def check_json():
     r_time = 0
 
     try:
-        dnam_pk = data.get('dnam_pk').strip()
-        account = Account.from_key(dnam_pk)
-        dnam_wallet = account.address
-        dark_gw, dark_map = get_bc_gateway(dnam_pk)
+        account, dnam_wallet, dark_gw, dark_map = check_account(data)
+        # dnam_pk = data.get('dnam_pk').strip()
+        # account = Account.from_key(dnam_pk)
+        # dnam_wallet = account.address
+        # dark_gw, dark_map = get_bc_gateway(dnam_pk)
     except AttributeError:
         # sem a chave
         erros.append("No private key identified")
     except Exception:
-        erros.append("Invalid private key")
+        erros.append("Invalid private key.")
     except ValueError:
         erros.append("Invalid private key")
 
+    if len(erros) > 0:
+        resp = {'erros' :  erros , 'params' : str(data)}
+        return jsonify(resp), 500
 
 
     processados = []
@@ -75,7 +88,8 @@ def check_json():
 
         item_data = {'oai_id' : oai_id , 'requested_url' : url}
 
-        if oai_id and is_valid_url(url):
+
+        if oai_id and ( is_valid_url(url) and url_exists(url,dark_map) == False ):
             try: 
                 ark_hash_obj = shared_utils.get_pid(dnam_wallet,dark_map)
                 ark_hash_hex_val = ark_hash_obj.hex()
@@ -91,7 +105,10 @@ def check_json():
                 item_data['error_desc'] = str(e)
                 nao_processados.append(item_data)
         else:
-            item_data['error'] = 'invalid url'
+            if not is_valid_url(url):
+                item_data['error'] = 'invalid url'
+            else:
+                item_data['error'] = 'URL already exists and set to other ark'
             nao_processados.append(item_data)
 
     end_time = time.time()
@@ -144,3 +161,126 @@ def check_json():
     return jsonify(response), 200
 
 
+
+@load_api.route('/update', methods=['post'])
+def update_metadata():
+    data = request.json
+    items = data.get('items', [])
+    erros = []
+    update_time = 0
+
+    try:
+        account, dnam_wallet, dark_gw, dark_map = check_account(data)
+    except AttributeError:
+        # sem a chave
+        erros.append("No private key identified")
+    except Exception:
+        erros.append("Invalid private key")
+    except ValueError:
+        erros.append("Invalid private key")
+
+    # caso tenha parametros invalidos
+    if len(erros) > 0:
+        resp = {'erros' :  erros , 'params' : str(data)}
+        return jsonify(resp), 500
+
+    processados = []
+    nao_processados = []
+    start_time = time.time()
+    for item in items:
+
+        # recuperar o pid
+        parametros_validos = True
+        try:
+            dark_id = item.get('dark_id')
+            if not dark_id:
+                item['error'] = 'No valid dark_id provided'
+                nao_processados.append(item)
+                parametros_validos = False
+        except ValueError as e:
+            erros.append('No dark_id provided')
+            item['error'] = 'No valid dark_id provided'
+            nao_processados.append(item)
+            parametros_validos = False
+        try:
+            update_url = item.get('url')
+            item['update_url'] = update_url
+            del item['url']
+
+            if not update_url:
+                item['error'] = 'No url provided'
+                nao_processados.append(item)
+                parametros_validos = False
+            if not is_valid_url(update_url):
+                item['error'] = 'invalid URL'
+                nao_processados.append(item)
+                parametros_validos = False
+
+            # chek whether the url already exists
+            if url_exists(update_url,dark_map):
+                item['error'] = 'URL already exists and set to other ark'
+                nao_processados.append(item)
+                parametros_validos = False
+
+        except ValueError as e:
+            item['error'] = 'No url provided'
+            nao_processados.append(item)
+            parametros_validos = False
+
+        if parametros_validos:
+            try:
+                dark_pid = dark_map.get_pid_by_ark(dark_id)
+            except Exception as e:
+                item['error'] = 'dark_id not found'
+                nao_processados.append(item)
+                break
+                # resp = jsonify({'status' : 'Unable to recovery (' + str(dark_id) + ')', 'block_chain_error' : str(e)},)
+                # resp_code = 500 # colocar um erro especifico?
+
+            item['update_url'] = update_url
+
+            if is_valid_url(update_url):
+                try:
+                    ark_hash_obj = dark_pid.pid_hash
+                    ark_hash_hex_val = ark_hash_obj.hex()
+                    item['ark_hash'] = ark_hash_hex_val
+                    tx = dark_map.async_set_url(ark_hash_obj,update_url)
+                    txr = tx.hex()
+                    item['tx_recipt'] = txr
+                    item['previous_url'] = dark_pid.external_url
+                    processados.append(item)
+                except Exception as e: 
+                    item['error'] = 'bockchain error'
+                    item['error_desc'] = str(e)
+                    nao_processados.append(item)
+            else:
+                item['error'] = 'invalid url'
+                nao_processados.append(item)
+                # print("aqui")
+
+
+    end_time = time.time()
+    update_time = end_time - start_time
+
+    response = {
+            "wallet_addr": account.address ,
+            "action": 'update',
+            "update_time" : update_time,
+            "timestamp": datetime.now().timestamp(),
+            "updated_pids": processados
+    }
+
+    if len(nao_processados) != 0:
+        response["not_updated_pids"] = nao_processados
+
+
+    # 
+    os.makedirs('logs', exist_ok=True)
+    timestamp = int(time.time())
+    random_digits = random.randint(000, 999)
+    filename = f"logs/update_{time.strftime('%Y-%m-%d')}_{dnam_wallet[2:]}_{timestamp}_{random_digits}.json"
+    with open(filename, 'w') as f:
+        json.dump(response, f, indent=4)
+
+    return jsonify(response), 200
+

app/util/bc.py

@@ -0,0 +1,7 @@
+from dark import DarkMap
+# from utils import is_valid_url
+
+def url_exists(url: str, dm:DarkMap) -> bool:
+    # print(url, dm.url_db.caller.exist(url))
+    return dm.url_db.caller.exist(url)
+

tests/README.md

@@ -0,0 +1 @@
+Use Ress Client (github)[https://github.com/Huachao/vscode-restclient], (market)[https://marketplace.visualstudio.com/items/?itemName=humao.rest-client] to test

tests/update.http

@@ -0,0 +1,98 @@
+### Variables 
+@apiurl = http://127.0.0.1:7000
+
+# note : rmber to change de ark for valid arks
+
+
+### load invalido 1
+POST {{apiurl}}/update HTTP/1.1
+Content-Type: application/json
+
+{
+    "dnam_pk": "R1",
+    "items": [ 
+        {"oai_id" : "a" , "url" : "invalida"},
+        {"oai_id" : "b" , "url" : "http://uol.com"},
+        {"oai_id" : "c" , "url" : "http://uol.com"},
+        {"oai_id" : "d" , "url" : "http://uol.com"}
+    ]
+
+}
+
+### load invalido 2
+POST {{apiurl}}/update HTTP/1.1
+Content-Type: application/json
+
+{
+    "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f",
+    "items": [ 
+        {"oai_id" : "a" , "url" : "invalida"}
+    ]
+
+}
+
+### load invalido 3
+POST {{apiurl}}/update HTTP/1.1
+Content-Type: application/json
+
+{
+    "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f",
+    "items": [ 
+        {"oai_id" : "a" , "url" : "invalida"},
+        {"dark_id" : "b" , "url" : "http://uol.com"}
+    ]
+
+}
+
+### load invalida 4
+POST {{apiurl}}/update HTTP/1.1
+Content-Type: application/json
+
+{
+    "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f",
+    "items": [ 
+        {"dark_id" : "8033/fkwf300000004h" , "url" : "http:/gov.com/"}
+    ]
+
+}
+
+### load valida 1
+POST {{apiurl}}/update HTTP/1.1
+Content-Type: application/json
+
+{
+    "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f",
+    "items": [ 
+        {"dark_id" : "8033/fkwf30000003gg" , "url" : "http://uol.com/up1"}
+    ]
+
+}
+
+
+### load valida 2
+POST {{apiurl}}/update HTTP/1.1
+Content-Type: application/json
+
+{
+    "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f",
+    "items": [ 
+        {"dark_id" : "8033/fkwf300000001n" , "url" : "http://nova.com/up1"},
+        {"dark_id" : "8033/fkwf3000000028" , "url" : "http://new.com/up1"},
+        {"dark_id" : "8033/fkwf300000004h" , "url" : "http://nueva.com/up1"}        
+    ]
+
+}
+
+### load invalida 4
+POST {{apiurl}}/update HTTP/1.1
+Content-Type: application/json
+
+{
+    "dnam_pk": "0xINVALIDA_NOT_VALID_INVALIDA_4d162f",
+    "items": [ 
+        {"dark_id" : "8033/fkwf300000004h" , "url" : "http:/gov.com/"},
+        {"dark_id" : "8033/fkwf3000000028" , "url" : "http://tudook.com/update"}
+    ]
+
+}
+