From 9c72b0b9f4c6ba2cd4d380000fbcd1fdfd43a40b Mon Sep 17 00:00:00 2001
From: luizmateuss <99288913+LuizMateuss@users.noreply.github.com>
Date: Sat, 3 Aug 2024 17:26:11 -0300
Subject: [PATCH] refact: salva outliers em arquivo csv na camada gold, separa
 envio para api em outro script

---
 etis/2_silver/.gitempty      |   0
 etis/3_gold/.gitempty        |   0
 requirements.txt             | Bin 374 -> 231 bytes
 scripts/deteccao_outliers.py |  32 +++++++++++++++++++++-----------
 scripts/enviar_outliers.py   |  23 +++++++++++++++++++++++
 5 files changed, 44 insertions(+), 11 deletions(-)
 create mode 100644 etis/2_silver/.gitempty
 create mode 100644 etis/3_gold/.gitempty
 create mode 100644 scripts/enviar_outliers.py

diff --git a/etis/2_silver/.gitempty b/etis/2_silver/.gitempty
new file mode 100644
index 0000000..e69de29
diff --git a/etis/3_gold/.gitempty b/etis/3_gold/.gitempty
new file mode 100644
index 0000000..e69de29
diff --git a/requirements.txt b/requirements.txt
index d30af55357fa77cdc37fdc9bf95db9ec3d3997c2..933478f68c0f05c7dc9dce578ac81b9b7b2bfe0d 100644
GIT binary patch
literal 231
zcmXv}K@P(p47=wqMFpl#JLEB}WhxQ{24Na~{X*9&7e}^ZXSSK#%OR>!-NYmm%&W=8
z(hy?ajCZ@uh+1f7+!9R3cL;S$jXh!%5|RmvDShLCR4O@-I59UuqPFz|jJvYY8pE&{
zvVV~h4<=G9#L|<DWWxthUXK0jCL!B*wS|r-t%Qc+zFjCe3ux!FT<`cpLe1X0r@mVK
E01iAv=l}o!

literal 374
zcmZvY-3o$05QWcm(4!=3QFPJ6m=y}9=1)`)Uwt!c(M8AzyK~NWX6N<HbyTN8rB2k2
zFI7yf6lqPRnro(9g)($K>Ol+DYB-&_t8hJ>VcsPUtKceiK$$t!Za2<qp^TnbdB;*>
zFH36a+#%akMhu-u$>hD$FG#PM;sAR<`AbV7Z6{)ju1697^)~LUb@iNN@`RiY>ILr6
l{k<!UyyRu}w@G_me*1b8-&RN(nZ5GO?9qmu+D<-uc>}^%I2ix{

diff --git a/scripts/deteccao_outliers.py b/scripts/deteccao_outliers.py
index 64fe44e..16c4ae2 100644
--- a/scripts/deteccao_outliers.py
+++ b/scripts/deteccao_outliers.py
@@ -5,7 +5,7 @@
 from statistics import median
 import requests
 import math
-
+import csv
 
 class BaseDetectorOutlier:
     def __init__(self, csv_path, epsilon=3):
@@ -146,17 +146,27 @@ def escrever_csv(self):
     load_dotenv()
     CAMINHO_BASE = f'{os.getenv("CAMINHO_SAIDA")}'
     outliers = []
-    for file in os.listdir(CAMINHO_BASE + '/2_silver/'):
+    silver_folder = os.listdir(CAMINHO_BASE + '/2_silver/')
+    for file in silver_folder:
+        if ".gitempty" == file:
+            continue
         test = BaseDetectorOutlier(CAMINHO_BASE + '/2_silver/' + file, 3)
         #test.descricao()
         
-        outliers += test.get_outliers()
-        
-        print('len: ', len(outliers))
-        for o in outliers:
-            print('out: ', o)
-            res = requests.post(f'{os.getenv("DOMINIO")}/Indicador/Add', json=o)
-            print('res: ', res.status_code)
-            print('res: ', res.text)
-        
+        outliers += test.get_outliers()   
     print(outliers)
+    print('len: ', len(outliers))
+
+    # Campos que serão usados como cabeçalhos no arquivo CSV
+    fieldnames = outliers[0].keys()
+
+    save_dir = os.path.join(f'{CAMINHO_BASE}/3_gold/')
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+    # Salvando como CSV
+    with open(f'{save_dir}outliers.csv', mode='w', newline='') as file:
+        writer = csv.DictWriter(file, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(outliers)
+
+
diff --git a/scripts/enviar_outliers.py b/scripts/enviar_outliers.py
new file mode 100644
index 0000000..1b6a718
--- /dev/null
+++ b/scripts/enviar_outliers.py
@@ -0,0 +1,23 @@
+import csv
+import requests
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+CAMINHO_BASE = f'{os.getenv("CAMINHO_SAIDA")}'
+
+dir = os.path.join(f'{CAMINHO_BASE}/3_gold/outliers.csv')
+
+with open(dir, mode='r', newline='') as file:
+    outliers = csv.DictReader(file)
+    for o in outliers:
+        print('out: ', o)
+        try:
+            response = requests.post(f'{os.getenv("DOMINIO")}/Indicador/Add', json=o)
+            if response.status_code == 200:
+                print(f"Sucesso: {response.json()}")
+            else:
+                print(f"Falha na requisição: {response.status_code}, {response.text}")
+        except Exception as e:
+            print('Erro: ', e)
+        
\ No newline at end of file