Skip to content

Commit b27c328

Browse files
woodthom2woodthom2
committed
Tests for staging deployment
1 parent 94647d8 commit b27c328

5 files changed

Lines changed: 570 additions & 0 deletions

File tree

‎tests/generate_remote_test.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,8 @@ sed s#http://localhost:8000#https://api.harmonydata.ac.uk#g local_tests/test_par
22
sed s#http://localhost:8000#https://api.harmonydata.ac.uk#g local_tests/test_match_local.py > remote_tests/test_match_remote.py
33
sed s#http://localhost:8000#https://api.harmonydata.ac.uk#g local_tests/test_match_mhc_local.py > remote_tests/test_match_mhc_remote.py
44
sed s#http://localhost:8000#https://api.harmonydata.ac.uk#g local_tests/test_different_frameworks_local.py > remote_tests/test_different_frameworks_remote.py
5+
sed s#http://localhost:8000#https://harmonystagingtmp.azurewebsites.net#g local_tests/test_parse_local.py > staging_tests/test_parse_remote.py
6+
sed s#http://localhost:8000#https://harmonystagingtmp.azurewebsites.net#g local_tests/test_match_local.py > staging_tests/test_match_remote.py
7+
sed s#http://localhost:8000#https://harmonystagingtmp.azurewebsites.net#g local_tests/test_match_mhc_local.py > staging_tests/test_match_mhc_remote.py
8+
sed s#http://localhost:8000#https://harmonystagingtmp.azurewebsites.net#g local_tests/test_different_frameworks_local.py > staging_tests/test_different_frameworks_remote.py
59
sed s#http://localhost:3000#https://harmonydata.ac.uk#g selenium_tests/test_selenium_local.py > selenium_tests/test_selenium_remote.py

‎tests/staging_tests/test_different_frameworks_staging.py‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
'''
2+
MIT License
3+
4+
Copyright (c) 2023 Ulster University (https://www.ulster.ac.uk).
5+
Project: Harmony (https://harmonydata.ac.uk)
6+
Maintainer: Thomas Wood (https://fastdatascience.com)
7+
8+
Permission is hereby granted, free of charge, to any person obtaining a copy
9+
of this software and associated documentation files (the "Software"), to deal
10+
in the Software without restriction, including without limitation the rights
11+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12+
copies of the Software, and to permit persons to whom the Software is
13+
furnished to do so, subject to the following conditions:
14+
15+
The above copyright notice and this permission notice shall be included in all
16+
copies or substantial portions of the Software.
17+
18+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24+
SOFTWARE.
25+
26+
'''
27+
28+
import json
29+
import unittest
30+
31+
import requests
32+
33+
headers = {
34+
'accept': 'application/json',
35+
'Content-Type': 'application/json',
36+
'Accept-Encoding': 'gzip, deflate, br',
37+
}
38+
39+
json_data_to_match_gad_7 = {
40+
'instruments': [
41+
{
42+
'file_id': 'fd60a9a64b1b4078a68f4bc06f20253c',
43+
'instrument_id': '7829ba96f48e4848abd97884911b6795',
44+
'instrument_name': 'GAD-7 English',
45+
'file_name': 'GAD-7 EN.pdf',
46+
'file_type': 'pdf',
47+
'file_section': 'GAD-7 English',
48+
'language': 'en',
49+
'questions': [
50+
{
51+
'question_no': '1',
52+
'question_intro': 'Over the last two weeks, how often have you been bothered by the following problems?',
53+
'question_text': 'Feeling nervous, anxious, or on edge',
54+
'options': [
55+
'Not at all',
56+
'Several days',
57+
'More than half the days',
58+
'Nearly every day',
59+
],
60+
'source_page': 0,
61+
},
62+
{
63+
'question_no': '2',
64+
'question_intro': 'Over the last two weeks, how often have you been bothered by the following problems?',
65+
'question_text': 'Not being able to stop or control worrying',
66+
'options': [
67+
'Not at all',
68+
'Several days',
69+
'More than half the days',
70+
'Nearly every day',
71+
],
72+
'source_page': 0,
73+
},
74+
],
75+
},
76+
{
77+
'file_id': 'fd60a9a64b1b4078a68f4bc06f20253c',
78+
'instrument_id': '7829ba96f48e4848abd97884911b6795',
79+
'instrument_name': 'GAD-7 Portuguese',
80+
'file_name': 'GAD-7 PT.pdf',
81+
'file_type': 'pdf',
82+
'file_section': 'GAD-7 Portuguese',
83+
'language': 'en',
84+
'questions': [
85+
{
86+
'question_no': '1',
87+
'question_intro': 'Durante as últimas 2 semanas, com que freqüência você foi incomodado/a pelos problemas abaixo?',
88+
'question_text': 'Sentir-se nervoso/a, ansioso/a ou muito tenso/a',
89+
'options': [
90+
'Nenhuma vez',
91+
'Vários dias',
92+
'Mais da metade dos dias',
93+
'Quase todos os dias',
94+
],
95+
'source_page': 0,
96+
},
97+
{
98+
'question_no': '2',
99+
'question_intro': 'Durante as últimas 2 semanas, com que freqüência você foi incomodado/a pelos problemas abaixo?',
100+
'question_text': ' Não ser capaz de impedir ou de controlar as preocupações',
101+
'options': [
102+
'Nenhuma vez',
103+
'Vários dias',
104+
'Mais da metade dos dias',
105+
'Quase todos os dias',
106+
],
107+
'source_page': 0,
108+
},
109+
],
110+
},
111+
],
112+
'query': 'anxiety',
113+
'parameters': {
114+
'framework': 'huggingface',
115+
'model': 'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2',
116+
},
117+
}
118+
119+
endpoint = 'https://harmonystagingtmp.azurewebsites.net/text/match'
120+
121+
122+
class TestDifferentFrameworks(unittest.TestCase):
123+
124+
def test_huggingface(self):
125+
response = requests.post(endpoint, headers=headers, json=json_data_to_match_gad_7)
126+
127+
print(response.json()["matches"][0][0])
128+
129+
self.assertLess(0.99, response.json()["matches"][0][0])
130+
131+
def test_google_nonexistent(self):
132+
payload_google = json.loads(json.dumps(json_data_to_match_gad_7))
133+
payload_google["parameters"]["framework"] = "google"
134+
135+
response = requests.post(endpoint, headers=headers, json=payload_google)
136+
137+
self.assertIn("model does not exist", response.text)
138+
139+
def test_google_real(self):
140+
payload_google = json.loads(json.dumps(json_data_to_match_gad_7))
141+
payload_google["parameters"]["framework"] = "google"
142+
payload_google["parameters"]["model"] = "textembedding-gecko@003"
143+
144+
response = requests.post(endpoint, headers=headers, json=payload_google)
145+
146+
self.assertLess(0.99, response.json()["matches"][0][0])
147+
148+
def test_google_different_from_huggingface(self):
149+
response_huggingface = requests.post(endpoint, headers=headers, json=json_data_to_match_gad_7)
150+
151+
payload_google = json.loads(json.dumps(json_data_to_match_gad_7))
152+
payload_google["parameters"]["framework"] = "google"
153+
payload_google["parameters"]["model"] = "textembedding-gecko@003"
154+
155+
response_google = requests.post(endpoint, headers=headers, json=payload_google)
156+
157+
abs_diff = abs(response_huggingface.json()["matches"][0][1] - response_google.json()["matches"][0][1])
158+
159+
print("Absolute difference", abs_diff)
160+
161+
self.assertLess(0.1, abs_diff)
162+
163+
def test_google_monolingual(self):
164+
payload_google = json.loads(json.dumps(json_data_to_match_gad_7))
165+
payload_google["parameters"]["framework"] = "google"
166+
payload_google["parameters"]["model"] = "textembedding-gecko@003"
167+
168+
response = requests.post(endpoint, headers=headers, json=payload_google)
169+
170+
self.assertGreater(0.92, response.json()["matches"][0][2])
171+
172+
def test_google_multilingual(self):
173+
payload_google = json.loads(json.dumps(json_data_to_match_gad_7))
174+
payload_google["parameters"]["framework"] = "google"
175+
payload_google["parameters"]["model"] = "textembedding-gecko-multilingual"
176+
177+
response = requests.post(endpoint, headers=headers, json=payload_google)
178+
179+
self.assertLess(0.92, response.json()["matches"][0][2])
180+
181+
def test_openai(self):
182+
payload_openai = json.loads(json.dumps(json_data_to_match_gad_7))
183+
payload_openai["parameters"]["framework"] = "openai"
184+
payload_openai["parameters"]["model"] = "text-embedding-3-large"
185+
186+
response = requests.post(endpoint, headers=headers, json=payload_openai)
187+
188+
self.assertLess(0.99, response.json()["matches"][0][0])
189+
190+
def test_azure_openai(self):
191+
payload_openai = json.loads(json.dumps(json_data_to_match_gad_7))
192+
payload_openai["parameters"]["framework"] = "azure_openai"
193+
payload_openai["parameters"]["model"] = "fds-text-embedding-3-large"
194+
195+
response = requests.post(endpoint, headers=headers, json=payload_openai)
196+
197+
self.assertLess(0.99, response.json()["matches"][0][0])
198+
199+
def test_azure_openai_same_as_vanilla_openai(self):
200+
payload_vanilla_openai = json.loads(json.dumps(json_data_to_match_gad_7))
201+
payload_vanilla_openai["parameters"]["framework"] = "openai"
202+
payload_vanilla_openai["parameters"]["model"] = "text-embedding-3-large"
203+
204+
vanilla_openai_response = requests.post(endpoint, headers=headers, json=payload_vanilla_openai)
205+
206+
payload_azure_openai = json.loads(json.dumps(json_data_to_match_gad_7))
207+
payload_azure_openai["parameters"]["framework"] = "azure_openai"
208+
payload_azure_openai["parameters"]["model"] = "fds-text-embedding-3-large"
209+
210+
azure_openai_response = requests.post(endpoint, headers=headers, json=payload_azure_openai)
211+
212+
self.assertGreater(0.01,
213+
abs(vanilla_openai_response.json()["matches"][0][0] -
214+
azure_openai_response.json()["matches"][0][0]))
215+
self.assertGreater(0.01,
216+
abs(vanilla_openai_response.json()["matches"][0][1] -
217+
azure_openai_response.json()["matches"][0][1]))
218+
self.assertGreater(0.01,
219+
abs(vanilla_openai_response.json()["matches"][0][2] -
220+
azure_openai_response.json()["matches"][0][2]))
221+
222+
223+
if __name__ == '__main__':
224+
unittest.main()

‎tests/staging_tests/test_match_mhc_staging.py‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
'''
2+
MIT License
3+
4+
Copyright (c) 2023 Ulster University (https://www.ulster.ac.uk).
5+
Project: Harmony (https://harmonydata.ac.uk)
6+
Maintainer: Thomas Wood (https://fastdatascience.com)
7+
8+
Permission is hereby granted, free of charge, to any person obtaining a copy
9+
of this software and associated documentation files (the "Software"), to deal
10+
in the Software without restriction, including without limitation the rights
11+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12+
copies of the Software, and to permit persons to whom the Software is
13+
furnished to do so, subject to the following conditions:
14+
15+
The above copyright notice and this permission notice shall be included in all
16+
copies or substantial portions of the Software.
17+
18+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24+
SOFTWARE.
25+
26+
'''
27+
28+
import sys
29+
import unittest
30+
31+
sys.path.append("../harmony/src")
32+
import requests
33+
34+
# mhc_questions, mhc_all_metadata, mhc_embeddings = helpers.get_mhc_embeddings()
35+
36+
37+
headers = {
38+
'accept': 'application/json',
39+
'Content-Type': 'application/json',
40+
'Accept-Encoding': 'gzip, deflate, br',
41+
}
42+
43+
json_data_to_match_gad_7 = {
44+
'instruments': [
45+
{
46+
'file_id': 'fd60a9a64b1b4078a68f4bc06f20253c',
47+
'instrument_id': '7829ba96f48e4848abd97884911b6795',
48+
'instrument_name': 'GAD-7 English',
49+
'file_name': 'GAD-7 EN.pdf',
50+
'file_type': 'pdf',
51+
'file_section': 'GAD-7 English',
52+
'language': 'en',
53+
'questions': [
54+
{
55+
'question_no': '1',
56+
'question_intro': 'Over the last two weeks, how often have you been bothered by the following problems?',
57+
'question_text': 'Feeling nervous, anxious, or on edge',
58+
'options': [
59+
'Not at all',
60+
'Several days',
61+
'More than half the days',
62+
'Nearly every day',
63+
],
64+
'source_page': 0,
65+
},
66+
{
67+
'question_no': '2',
68+
'question_intro': 'Over the last two weeks, how often have you been bothered by the following problems?',
69+
'question_text': 'Not being able to stop or control worrying',
70+
'options': [
71+
'Not at all',
72+
'Several days',
73+
'More than half the days',
74+
'Nearly every day',
75+
],
76+
'source_page': 0,
77+
},
78+
],
79+
},
80+
{
81+
'file_id': 'fd60a9a64b1b4078a68f4bc06f20253c',
82+
'instrument_id': '7829ba96f48e4848abd97884911b6795',
83+
'instrument_name': 'GAD-7 Portuguese',
84+
'file_name': 'GAD-7 PT.pdf',
85+
'file_type': 'pdf',
86+
'file_section': 'GAD-7 Portuguese',
87+
'language': 'en',
88+
'questions': [
89+
{
90+
'question_no': '1',
91+
'question_intro': 'Durante as últimas 2 semanas, com que freqüência você foi incomodado/a pelos problemas abaixo?',
92+
'question_text': 'Sentir-se nervoso/a, ansioso/a ou muito tenso/a',
93+
'options': [
94+
'Nenhuma vez',
95+
'Vários dias',
96+
'Mais da metade dos dias',
97+
'Quase todos os dias',
98+
],
99+
'source_page': 0,
100+
},
101+
{
102+
'question_no': '2',
103+
'question_intro': 'Durante as últimas 2 semanas, com que freqüência você foi incomodado/a pelos problemas abaixo?',
104+
'question_text': ' Não ser capaz de impedir ou de controlar as preocupações',
105+
'options': [
106+
'Nenhuma vez',
107+
'Vários dias',
108+
'Mais da metade dos dias',
109+
'Quase todos os dias',
110+
],
111+
'source_page': 0,
112+
},
113+
],
114+
},
115+
],
116+
'query': 'anxiety',
117+
'parameters': {
118+
'framework': 'huggingface',
119+
'model': 'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2',
120+
},
121+
}
122+
123+
endpoint = 'https://harmonystagingtmp.azurewebsites.net/text/match'
124+
125+
response = requests.post(endpoint, headers=headers, json=json_data_to_match_gad_7)
126+
127+
128+
class TestMatchMhcLocal(unittest.TestCase):
129+
130+
def test_match_mhc_local(self):
131+
response_dict = response.json()
132+
self.assertEqual(4, len(response_dict["questions"]))
133+
self.assertEqual("anxiety", response_dict["questions"][0]["topics_auto"][0])
134+
self.assertEqual("anxiety", list(response_dict["questions"][0]["topics_strengths"])[0])
135+
136+
137+
if __name__ == '__main__':
138+
unittest.main()

0 commit comments

Comments
 (0)