[RESEND PATCH v1 3/4] index: Integrate vn-stats.json with the scraper

[Taufiq Pohan <m.taufiq30s@hotmail.com>]

vn-stats.json saves the last scraped VN ID from VNDB. Integrate this
state to our scraper.

Signed-off-by: Ammar Faizi <ammarfaizi2@gnuweeb.org>
Co-authored-by: Ammar Faizi <ammarfaizi2@gnuweeb.org>
Signed-off-by: Aldy Prastyo <dipras75@gmail.com>
Co-authored-by: Aldy Prastyo <dipras75@gmail.com>
Signed-off-by: Taufiq Pohan <m.taufiq30s@hotmail.com>
---
 index.js | 82 +++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 58 insertions(+), 24 deletions(-)

diff --git a/index.js b/index.js
index 9ceee76..513ffad 100644
--- a/index.js
+++ b/index.js
@@ -7,21 +7,26 @@ import fs from 'fs';
 
 config();
 
-mongoose.connect(process.env.MONGODB_URI, {
-	useNewUrlParser: true,
-	useUnifiedTopology: true
-});
-
-const init_db = () =>
-	mongoose.connection
-		.on('error', (error) => console.error(error))
-		.once('open', () => console.log('Database Connected'));
-
 async function get_vn_by_code(code)
 {
 	return await vndb.query(`get vn details,basic,stats (id = ${code})`);
 }
 
+async function get_number_of_vndb_vns()
+{
+	let res = await vndb.query("dbstats");
+
+	if (!("vn" in res))
+		throw Error("Error, vndb malformed response");
+
+	return res.vn;
+}
+
+async function get_number_of_our_vns()
+{
+	return await model.countDocuments();
+}
+
 async function insert_to_db(result)
 {
 	const body = {
@@ -65,32 +70,61 @@ function save_last_id(id)
 
 function get_last_id()
 {
-	if (fs.existsSync('./vn-stats.json')) {
-		const jsonVal = require('./vn-stats.json');
-		return jsonVal['last_vn_id'];
-	}
+	if (!fs.existsSync('./vn-stats.json'))
+		return 1;
 
-	return 1;
+	const jsonVal = fs.readFileSync('./vn-stats.json');
+	let ret = JSON.parse(jsonVal);
+	if (!("last_vn_id" in ret) || isNaN(ret.last_vn_id))
+		return 1;
+
+	return ret.last_vn_id
+}
+
+function sleep(ms)
+{
+	return new Promise((resolve) => {
+		setTimeout(resolve, ms);
+	});
 }
 
-async function main()
+async function start_scrape()
 {
-	init_db();
+	let i = get_last_id() + 1;
 
-	let code = 40029;
-	let i;
+	while (true) {
+		let nr_vns_ours = get_number_of_our_vns();
+		let nr_vns_vndb = get_number_of_vndb_vns();
+
+		if (nr_vns_vndb == nr_vns_ours)
+			break;
 
-	i = code - 5;
-	while (i++) {
 		console.log(`Scraping VN ${i}...`);
 		let ret = await scrape_vn_and_save_to_db(i);
 		if (!ret)
 			break;
+
 		console.log(`Successfully scraped VN ${i}`);
+		save_last_id(i);
+		i++;
+		await sleep(1000);
 	}
-	console.log(`Last VN ID is ${code}`);
-	save_last_id(i);
 	process.exit();
 }
 
-main();
\ No newline at end of file
+function main()
+{
+	mongoose.connect(process.env.MONGODB_URI, {
+		useNewUrlParser: true,
+		useUnifiedTopology: true
+	});
+
+	mongoose.connection
+		.on('error', (error) => console.error(error))
+		.once('open', async function () {
+			console.log('Database Connected');
+			await start_scrape();
+		});
+}
+
+main();
-- 
Taufiq Pohan