web scrapping

imrahulkumar · imrahulkumar · commit 1fb672e11593 · 2020-11-13T02:22:39.000+05:30
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -16,6 +16,7 @@
   "dependencies": {
     "@types/axios": "^0.14.0",
     "@types/body-parser": "^1.19.0",
+    "@types/cheerio": "^0.22.22",
     "@types/express": "^4.17.8",
     "@types/joi": "^14.3.4",
     "@types/jsonwebtoken": "^8.5.0",
@@ -24,9 +25,11 @@
     "@types/node": "^14.14.2",
     "@types/node-schedule": "^1.3.1",
     "@types/nodemailer": "^6.4.0",
+    "@types/request": "^2.48.5",
     "axios": "^0.21.0",
     "bcrypt": "^5.0.0",
     "body-parser": "^1.19.0",
+    "cheerio": "^1.0.0-rc.3",
     "emailjs": "^3.3.0",
     "emailjs-com": "^2.6.3",
     "express": "^4.17.1",
@@ -40,7 +43,8 @@
     "nodemailer": "^6.4.14",
     "nodemailer-sendgrid-transport": "^0.2.0",
     "nodemon": "^2.0.6",
+    "request": "^2.88.2",
     "ts-node": "^9.0.0",
     "typescript": "^4.0.3"
   }
-}
+}
diff --git a/readme.md b/readme.md
@@ -137,6 +137,17 @@ npm install --save @types/joi
   - npm i loadtest
   -loadtest [-n requests] [-c concurrency] [-k] URL
   -loadtest -n 100 -c 20 -k "https://node-ts-blog.herokuapp.com/api/user/login"
+
+
+## For Web Scraping
+
+  - To get the html data from the website use "Request - Simplified HTTP client"
+  - npm i request
+  - npm i @types/request
+
+  - To get the specific data from the html use "cheerio"
+  - npm i cheerio
+  - npm i @types/cheerio
   
 
 
diff --git a/src/controllers/UserControllers.ts b/src/controllers/UserControllers.ts
@@ -5,6 +5,9 @@ import { getEnvironmentVariable } from '../environments/env';
 import { Emailjs } from '../utils/Emailjs';
 import { EmailTemplate } from '../utils/TemplateEmailjs';
 
+import * as Cheerio from 'cheerio';
+import * as Request from 'request';
+
 export class UserController {
 
     static async signup(req, res, next) {
@@ -178,10 +181,30 @@ export class UserController {
 
     // Multi Field Indexing
     static async test(req, res, next) {
-        const user = await User.find({ email: 'rahul.k@mailinator.com',
-                                       password:'$2b$10$5mRQr1IU5SjQfDR6WeKNJOoLcZB9IiFg0i81tdgSWReXRKV7Cm5VG"' })
-                          .setOptions({explain:'executionStats'});
-                          res.send(user)
+        const user = await User.find({
+            email: 'rahul.k@mailinator.com',
+            password: '$2b$10$5mRQr1IU5SjQfDR6WeKNJOoLcZB9IiFg0i81tdgSWReXRKV7Cm5VG"'
+        })
+            .setOptions({ explain: 'executionStats' });
+        res.send(user)
+    }
+
+
+    static async webScraping(req, res, next) {
+        Request('https://webscraper.io/test-sites/e-commerce/allinone', ((error, response, html) => {
+            if (!error && response.statusCode == 200) {
+                const $ = Cheerio.load(html);
+                const data = [];
+                $('.thumbnail').each((index, element) => {
+                    const image = $(element).find('.img-responsive').attr('src');
+                    const title = $(element).find('.title').text();
+                    const description = $(element).find('.description').text();
+                    const price = $(element).find('.price').text();
+                    data.push({title: title, image: image, description: description, price: price});
+                });
+                res.send(data);
+            }
+        }))
     }
 
 
diff --git a/src/routers/UserRouter.ts b/src/routers/UserRouter.ts
@@ -29,6 +29,9 @@ class UserRouter {
 
         //To check the index
         this.router.get('/test',UserController.test)
+
+        //To check web-scraping
+        this.router.get('/web-scraping',UserController.webScraping)
    
     }
     postRoutes() {

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,9 @@ class UserRouter {`
`29`	`29`
`30`	`30`	`//To check the index`
`31`	`31`	`this.router.get('/test',UserController.test)`
	`32`	`+`
	`33`	`+ //To check web-scraping`
	`34`	`+ this.router.get('/web-scraping',UserController.webScraping)`
`32`	`35`
`33`	`36`	`}`
`34`	`37`	`postRoutes() {`