[RESEND PATCH v1 1/4] Add vndb-api, mongose, and dotenv module

[RESEND PATCH v1 1/4] Add vndb-api, mongose, and dotenv module

[Taufiq Pohan]

For initial project.

Signed-off-by: Taufiq Pohan <m.taufiq30s@hotmail.com>
---
 .gitignore   |  3 +++
 index.js     |  4 ++++
 package.json | 14 ++++++++++++++
 3 files changed, 21 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 index.js
 create mode 100644 package.json

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..b855cbc
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+.env
+node_modules/
+yarn.lock
\ No newline at end of file
diff --git a/index.js b/index.js
new file mode 100644
index 0000000..a3dadfb
--- /dev/null
+++ b/index.js
@@ -0,0 +1,4 @@
+import VNDB from "vndb-api";
+
+const vndb = new VNDB('atri_api');
+
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..7ef6a42
--- /dev/null
+++ b/package.json
@@ -0,0 +1,14 @@
+{
+  "name": "vndb_scraper",
+  "version": "1.0.0",
+  "description": "VNDB Scrapper for ATRI",
+  "main": "index.js",
+  "repository": "git@github.com:vnlx2/vndb_scraper.git",
+  "author": "Taufiq Pohan <m.taufiq30s@hotmail.com>",
+  "license": "GPLv2",
+  "dependencies": {
+    "dotenv": "^16.0.3",
+    "mongose": "^0.0.2-security",
+    "vndb-api": "^1.0.3"
+  }
+}
-- 
Taufiq Pohan

[RESEND PATCH v1 2/4] Initial VNDB scraper and storage management

[Taufiq Pohan]

Signed-off-by: Ammar Faizi <ammarfaizi2@gnuweeb.org>
Co-authored-by: Ammar Faizi <ammarfaizi2@gnuweeb.org>
Signed-off-by: Aldy Prastyo <dipras75@gmail.com>
Co-authored-by: Aldy Prastyo <dipras75@gmail.com>
Signed-off-by: Taufiq Pohan <m.taufiq30s@hotmail.com>
---
 .gitignore   |  7 ++--
 VNDBModel.js | 34 +++++++++++++++++++
 index.js     | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 package.json |  3 ++
 4 files changed, 135 insertions(+), 5 deletions(-)
 create mode 100644 VNDBModel.js

diff --git a/.gitignore b/.gitignore
index b855cbc..29ea801 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
-.env
-node_modules/
-yarn.lock
\ No newline at end of file
+/.env
+/node_modules/
+/yarn.lock
+/vn-stats.json
\ No newline at end of file
diff --git a/VNDBModel.js b/VNDBModel.js
new file mode 100644
index 0000000..4ba11a0
--- /dev/null
+++ b/VNDBModel.js
@@ -0,0 +1,34 @@
+import mongoose from "mongoose";
+
+// Schema
+
+const VisualNovel = mongoose.Schema({
+    code: {
+        type: String,
+        required: true
+    },
+    title: {
+        type: String,
+        required: true
+    },
+    alias: {
+        type: String
+    },
+    length: {
+        type: Number
+    },
+    rating: {
+        type: Number,
+    },
+    description: {
+        type: String,
+        required: true
+    },
+    image: {
+        type: String
+    }
+}, {
+    timestamps: true
+});
+
+export default mongoose.model('vndb', VisualNovel);
\ No newline at end of file
diff --git a/index.js b/index.js
index a3dadfb..9ceee76 100644
--- a/index.js
+++ b/index.js
@@ -1,4 +1,96 @@
-import VNDB from "vndb-api";
-
+import VNDB from 'vndb-api';
 const vndb = new VNDB('atri_api');
+import mongoose from "mongoose";
+import { config } from "dotenv";
+import model from './VNDBModel.js';
+import fs from 'fs';
+
+config();
+
+mongoose.connect(process.env.MONGODB_URI, {
+	useNewUrlParser: true,
+	useUnifiedTopology: true
+});
+
+const init_db = () =>
+	mongoose.connection
+		.on('error', (error) => console.error(error))
+		.once('open', () => console.log('Database Connected'));
+
+async function get_vn_by_code(code)
+{
+	return await vndb.query(`get vn details,basic,stats (id = ${code})`);
+}
+
+async function insert_to_db(result)
+{
+	const body = {
+		code: result.id,
+		title: result.title,
+		alias: result.alias,
+		length: result.length,
+		rating: result.rating,
+		description: result.image,
+		image: result.image
+	};
+	const response = await model(body);
+	await response.save();
+}
+
+async function scrape_vn_and_save_to_db(code)
+{
+	const result = await get_vn_by_code(code);
+	if (!result) {
+		console.log("Internal error");
+		return false;
+	}
+
+	if (result.items.length == 0) {
+		console.log(`VN ${code} is not found`);
+		return false;
+	}
+
+	insert_to_db(result.items[0]);
+	return true;
+}
+
+function save_last_id(id)
+{
+	const jsonVal = {
+		last_vn_id: id
+	};
+	fs.writeFileSync('vn-stats.json', JSON.stringify(jsonVal)+"\n");
+	return true;
+}
+
+function get_last_id()
+{
+	if (fs.existsSync('./vn-stats.json')) {
+		const jsonVal = require('./vn-stats.json');
+		return jsonVal['last_vn_id'];
+	}
+
+	return 1;
+}
+
+async function main()
+{
+	init_db();
+
+	let code = 40029;
+	let i;
+
+	i = code - 5;
+	while (i++) {
+		console.log(`Scraping VN ${i}...`);
+		let ret = await scrape_vn_and_save_to_db(i);
+		if (!ret)
+			break;
+		console.log(`Successfully scraped VN ${i}`);
+	}
+	console.log(`Last VN ID is ${code}`);
+	save_last_id(i);
+	process.exit();
+}
 
+main();
\ No newline at end of file
diff --git a/package.json b/package.json
index 7ef6a42..e5e9912 100644
--- a/package.json
+++ b/package.json
@@ -2,12 +2,15 @@
   "name": "vndb_scraper",
   "version": "1.0.0",
   "description": "VNDB Scrapper for ATRI",
+  "type": "module",
   "main": "index.js",
   "repository": "git@github.com:vnlx2/vndb_scraper.git",
   "author": "Taufiq Pohan <m.taufiq30s@hotmail.com>",
   "license": "GPLv2",
   "dependencies": {
     "dotenv": "^16.0.3",
+    "fs": "^0.0.1-security",
+    "mongoose": "^6.7.3",
     "mongose": "^0.0.2-security",
     "vndb-api": "^1.0.3"
   }
-- 
Taufiq Pohan

[RESEND PATCH v1 3/4] index: Integrate vn-stats.json with the scraper

[Taufiq Pohan]

vn-stats.json saves the last scraped VN ID from VNDB. Integrate this
state to our scraper.

Signed-off-by: Ammar Faizi <ammarfaizi2@gnuweeb.org>
Co-authored-by: Ammar Faizi <ammarfaizi2@gnuweeb.org>
Signed-off-by: Aldy Prastyo <dipras75@gmail.com>
Co-authored-by: Aldy Prastyo <dipras75@gmail.com>
Signed-off-by: Taufiq Pohan <m.taufiq30s@hotmail.com>
---
 index.js | 82 +++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 58 insertions(+), 24 deletions(-)

diff --git a/index.js b/index.js
index 9ceee76..513ffad 100644
--- a/index.js
+++ b/index.js
@@ -7,21 +7,26 @@ import fs from 'fs';
 
 config();
 
-mongoose.connect(process.env.MONGODB_URI, {
-	useNewUrlParser: true,
-	useUnifiedTopology: true
-});
-
-const init_db = () =>
-	mongoose.connection
-		.on('error', (error) => console.error(error))
-		.once('open', () => console.log('Database Connected'));
-
 async function get_vn_by_code(code)
 {
 	return await vndb.query(`get vn details,basic,stats (id = ${code})`);
 }
 
+async function get_number_of_vndb_vns()
+{
+	let res = await vndb.query("dbstats");
+
+	if (!("vn" in res))
+		throw Error("Error, vndb malformed response");
+
+	return res.vn;
+}
+
+async function get_number_of_our_vns()
+{
+	return await model.countDocuments();
+}
+
 async function insert_to_db(result)
 {
 	const body = {
@@ -65,32 +70,61 @@ function save_last_id(id)
 
 function get_last_id()
 {
-	if (fs.existsSync('./vn-stats.json')) {
-		const jsonVal = require('./vn-stats.json');
-		return jsonVal['last_vn_id'];
-	}
+	if (!fs.existsSync('./vn-stats.json'))
+		return 1;
 
-	return 1;
+	const jsonVal = fs.readFileSync('./vn-stats.json');
+	let ret = JSON.parse(jsonVal);
+	if (!("last_vn_id" in ret) || isNaN(ret.last_vn_id))
+		return 1;
+
+	return ret.last_vn_id
+}
+
+function sleep(ms)
+{
+	return new Promise((resolve) => {
+		setTimeout(resolve, ms);
+	});
 }
 
-async function main()
+async function start_scrape()
 {
-	init_db();
+	let i = get_last_id() + 1;
 
-	let code = 40029;
-	let i;
+	while (true) {
+		let nr_vns_ours = get_number_of_our_vns();
+		let nr_vns_vndb = get_number_of_vndb_vns();
+
+		if (nr_vns_vndb == nr_vns_ours)
+			break;
 
-	i = code - 5;
-	while (i++) {
 		console.log(`Scraping VN ${i}...`);
 		let ret = await scrape_vn_and_save_to_db(i);
 		if (!ret)
 			break;
+
 		console.log(`Successfully scraped VN ${i}`);
+		save_last_id(i);
+		i++;
+		await sleep(1000);
 	}
-	console.log(`Last VN ID is ${code}`);
-	save_last_id(i);
 	process.exit();
 }
 
-main();
\ No newline at end of file
+function main()
+{
+	mongoose.connect(process.env.MONGODB_URI, {
+		useNewUrlParser: true,
+		useUnifiedTopology: true
+	});
+
+	mongoose.connection
+		.on('error', (error) => console.error(error))
+		.once('open', async function () {
+			console.log('Database Connected');
+			await start_scrape();
+		});
+}
+
+main();
-- 
Taufiq Pohan

[RESEND PATCH v1 4/4] .gitignore: Add *.patch file to .gitingore

[Taufiq Pohan]

From: Ammar Faizi <ammarfaizi2@gnuweeb.org>

We usually generate patch file in the same directory with the
project. Ignore the generated patch file.

Signed-off-by: Ammar Faizi <ammarfaizi2@gnuweeb.org>
Signed-off-by: Taufiq Pohan <m.taufiq30s@hotmail.com>
---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 29ea801..477063a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 /.env
 /node_modules/
 /yarn.lock
-/vn-stats.json
\ No newline at end of file
+/vn-stats.json
+*.patch
\ No newline at end of file
-- 
Taufiq Pohan

Re: [RESEND PATCH v1 1/4] Add vndb-api, mongose, and dotenv module

[Ammar Faizi]

On Sun, 27 Nov 2022 02:37:21 +0700, Taufiq Pohan wrote:
> For initial project.
> 
> 

Applied, thanks!

[1/4] Add vndb-api, mongose, and dotenv module
      commit: 5c5380be2e4970d21729e1014a92aea25895d541
[2/4] Initial VNDB scraper and storage management
      commit: 8487845a100abf116a3ea0b2a3002f6813613ceb
[3/4] index: Integrate vn-stats.json with the scraper
      commit: 4784bb8e02e4bc7e650885b57128d6584cdaca14
[4/4] .gitignore: Add *.patch file to .gitingore
      commit: 6f43eec419f11a4850c36ec8a046a0425e38b1a2

Best regards,
-- 
Ammar Faizi <ammarfaizi2@gnuweeb.org>