[RESEND PATCH v1 2/4] Initial VNDB scraper and storage management

[Taufiq Pohan <m.taufiq30s@hotmail.com>]

Signed-off-by: Ammar Faizi <ammarfaizi2@gnuweeb.org>
Co-authored-by: Ammar Faizi <ammarfaizi2@gnuweeb.org>
Signed-off-by: Aldy Prastyo <dipras75@gmail.com>
Co-authored-by: Aldy Prastyo <dipras75@gmail.com>
Signed-off-by: Taufiq Pohan <m.taufiq30s@hotmail.com>
---
 .gitignore   |  7 ++--
 VNDBModel.js | 34 +++++++++++++++++++
 index.js     | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 package.json |  3 ++
 4 files changed, 135 insertions(+), 5 deletions(-)
 create mode 100644 VNDBModel.js

diff --git a/.gitignore b/.gitignore
index b855cbc..29ea801 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
-.env
-node_modules/
-yarn.lock
\ No newline at end of file
+/.env
+/node_modules/
+/yarn.lock
+/vn-stats.json
\ No newline at end of file
diff --git a/VNDBModel.js b/VNDBModel.js
new file mode 100644
index 0000000..4ba11a0
--- /dev/null
+++ b/VNDBModel.js
@@ -0,0 +1,34 @@
+import mongoose from "mongoose";
+
+// Schema
+
+const VisualNovel = mongoose.Schema({
+    code: {
+        type: String,
+        required: true
+    },
+    title: {
+        type: String,
+        required: true
+    },
+    alias: {
+        type: String
+    },
+    length: {
+        type: Number
+    },
+    rating: {
+        type: Number,
+    },
+    description: {
+        type: String,
+        required: true
+    },
+    image: {
+        type: String
+    }
+}, {
+    timestamps: true
+});
+
+export default mongoose.model('vndb', VisualNovel);
\ No newline at end of file
diff --git a/index.js b/index.js
index a3dadfb..9ceee76 100644
--- a/index.js
+++ b/index.js
@@ -1,4 +1,96 @@
-import VNDB from "vndb-api";
-
+import VNDB from 'vndb-api';
 const vndb = new VNDB('atri_api');
+import mongoose from "mongoose";
+import { config } from "dotenv";
+import model from './VNDBModel.js';
+import fs from 'fs';
+
+config();
+
+mongoose.connect(process.env.MONGODB_URI, {
+	useNewUrlParser: true,
+	useUnifiedTopology: true
+});
+
+const init_db = () =>
+	mongoose.connection
+		.on('error', (error) => console.error(error))
+		.once('open', () => console.log('Database Connected'));
+
+async function get_vn_by_code(code)
+{
+	return await vndb.query(`get vn details,basic,stats (id = ${code})`);
+}
+
+async function insert_to_db(result)
+{
+	const body = {
+		code: result.id,
+		title: result.title,
+		alias: result.alias,
+		length: result.length,
+		rating: result.rating,
+		description: result.image,
+		image: result.image
+	};
+	const response = await model(body);
+	await response.save();
+}
+
+async function scrape_vn_and_save_to_db(code)
+{
+	const result = await get_vn_by_code(code);
+	if (!result) {
+		console.log("Internal error");
+		return false;
+	}
+
+	if (result.items.length == 0) {
+		console.log(`VN ${code} is not found`);
+		return false;
+	}
+
+	insert_to_db(result.items[0]);
+	return true;
+}
+
+function save_last_id(id)
+{
+	const jsonVal = {
+		last_vn_id: id
+	};
+	fs.writeFileSync('vn-stats.json', JSON.stringify(jsonVal)+"\n");
+	return true;
+}
+
+function get_last_id()
+{
+	if (fs.existsSync('./vn-stats.json')) {
+		const jsonVal = require('./vn-stats.json');
+		return jsonVal['last_vn_id'];
+	}
+
+	return 1;
+}
+
+async function main()
+{
+	init_db();
+
+	let code = 40029;
+	let i;
+
+	i = code - 5;
+	while (i++) {
+		console.log(`Scraping VN ${i}...`);
+		let ret = await scrape_vn_and_save_to_db(i);
+		if (!ret)
+			break;
+		console.log(`Successfully scraped VN ${i}`);
+	}
+	console.log(`Last VN ID is ${code}`);
+	save_last_id(i);
+	process.exit();
+}
 
+main();
\ No newline at end of file
diff --git a/package.json b/package.json
index 7ef6a42..e5e9912 100644
--- a/package.json
+++ b/package.json
@@ -2,12 +2,15 @@
   "name": "vndb_scraper",
   "version": "1.0.0",
   "description": "VNDB Scrapper for ATRI",
+  "type": "module",
   "main": "index.js",
   "repository": "git@github.com:vnlx2/vndb_scraper.git",
   "author": "Taufiq Pohan <m.taufiq30s@hotmail.com>",
   "license": "GPLv2",
   "dependencies": {
     "dotenv": "^16.0.3",
+    "fs": "^0.0.1-security",
+    "mongoose": "^6.7.3",
     "mongose": "^0.0.2-security",
     "vndb-api": "^1.0.3"
   }
-- 
Taufiq Pohan