scrapegraph-js/integration_test.ts at main · ScrapeGraphAI/scrapegraph-js · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
import {
	type CreditsResponse,
	type HealthResponse,
	type MarkdownifyResponse,
	type ScrapeResponse,
	type SearchScraperResponse,
	type SitemapResponse,
	type SmartScraperResponse,
	checkHealth,
	getCredits,
	markdownify,
	scrape,
	searchScraper,
	sitemap,
	smartScraper,
} from "./src/index.js";

const maybeKey = process.env.SGAI_API_KEY;
if (!maybeKey) {
	console.error("Set SGAI_API_KEY env var");
	process.exit(1);
}
const apiKey: string = maybeKey;

function assert(condition: boolean, msg: string) {
	if (!condition) {
		console.error(`FAIL: ${msg}`);
		process.exit(1);
	}
}

function logResult(name: string, data: unknown) {
	console.log(`\n=== ${name} ===`);
	console.log(JSON.stringify(data, null, 2));
}

async function testHealth() {
	const res = await checkHealth(apiKey);
	logResult("checkHealth", res);
	assert(res.status === "success", "health status should be success");
	const d = res.data as HealthResponse;
	assert(typeof d.status === "string", "health.status should be string");
}

async function testCredits() {
	const res = await getCredits(apiKey);
	logResult("getCredits", res);
	assert(res.status === "success", "credits status should be success");
	const d = res.data as CreditsResponse;
	assert(typeof d.remaining_credits === "number", "remaining_credits should be number");
	assert(typeof d.total_credits_used === "number", "total_credits_used should be number");
}

async function testSmartScraper() {
	const res = await smartScraper(apiKey, {
		user_prompt: "Extract the page title and description",
		website_url: "https://example.com",
	});
	logResult("smartScraper", res);
	assert(res.status === "success", "smartScraper status should be success");
	const d = res.data as SmartScraperResponse;
	assert(typeof d.request_id === "string", "request_id should be string");
	assert(typeof d.status === "string", "status should be string");
	assert(typeof d.website_url === "string", "website_url should be string");
	assert(typeof d.user_prompt === "string", "user_prompt should be string");
	assert(d.result !== undefined, "result should exist");
}

async function testSearchScraper() {
	const res = await searchScraper(apiKey, {
		user_prompt: "What is the capital of France?",
	});
	logResult("searchScraper", res);
	assert(res.status === "success", "searchScraper status should be success");
	const d = res.data as SearchScraperResponse;
	assert(typeof d.request_id === "string", "request_id should be string");
	assert(typeof d.user_prompt === "string", "user_prompt should be string");
	assert(Array.isArray(d.reference_urls), "reference_urls should be array");
	assert(
		d.result !== undefined || d.markdown_content !== undefined,
		"result or markdown_content should exist",
	);
}

async function testMarkdownify() {
	const res = await markdownify(apiKey, {
		website_url: "https://example.com",
	});
	logResult("markdownify", res);
	assert(res.status === "success", "markdownify status should be success");
	const d = res.data as MarkdownifyResponse;
	assert(typeof d.request_id === "string", "request_id should be string");
	assert(typeof d.website_url === "string", "website_url should be string");
	assert(typeof d.result === "string" || d.result === null, "result should be string or null");
}

async function testScrape() {
	const res = await scrape(apiKey, {
		website_url: "https://example.com",
	});
	logResult("scrape", res);
	assert(res.status === "success", "scrape status should be success");
	const d = res.data as ScrapeResponse;
	assert(typeof d.scrape_request_id === "string", "scrape_request_id should be string");
	assert(typeof d.html === "string", "html should be string");
	assert(typeof d.status === "string", "status should be string");
}

async function testSitemap() {
	const res = await sitemap(apiKey, {
		website_url: "https://scrapegraphai.com",
	});
	logResult("sitemap", res);
	assert(res.status === "success", "sitemap status should be success");
	const d = res.data as SitemapResponse;
	assert(typeof d.request_id === "string", "request_id should be string");
	assert(Array.isArray(d.urls), "urls should be array");
}

console.log("Running API battle tests...\n");

await testHealth();
await testCredits();
await testSmartScraper();
await testSearchScraper();
await testMarkdownify();
await testScrape();
await testSitemap();

console.log("\nAll tests passed.");