Skip to content

Commit ffad4c5

Browse files
VLabStudioVLabStudio
committed
Added project files from Make your Own Web Crawler, Downloading Files using Node.js and Sending SMS using Node.js
1 parent e37187a commit ffad4c5

12 files changed

Lines changed: 419 additions & 0 deletions

File tree

‎Downloading Files using Node.js/app.js‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
const fs = require("fs");
2+
const download = require("download");
3+
4+
(async function () {
5+
6+
download("https://i.ytimg.com/vi/Oxj38vkwmBs/mqdefault.jpg", "dist").then(() => {
7+
console.log("done!");
8+
});
9+
10+
// const data = await download("https://i.ytimg.com/vi/Oxj38vkwmBs/mqdefault.jpg");
11+
// fs.writeFileSync("dist/foo2.jpg", data);
12+
// console.log("done!");
13+
14+
// await download("https://i.ytimg.com/vi/Oxj38vkwmBs/mqdefault.jpg", "dist");
15+
// console.log("done!");
16+
17+
// await download("https://codeload.github.com/VLabStudio/Tutorials/zip/master", "dist", { extract: true }); // filename
18+
// console.log("done!");
19+
20+
})();

‎Downloading Files using Node.js/package.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"name": "download-the-web",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "app.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"keywords": [],
10+
"author": "",
11+
"license": "MIT",
12+
"dependencies": {
13+
"download": "^7.1.0"
14+
}
15+
}

‎Make your Own Web Crawler/app.js‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Import dependencies
2+
const Crawler = require("crawler");
3+
const URL = require("url");
4+
const fs = require("fs");
5+
6+
// Make a new crawler
7+
const crawler = new Crawler({
8+
maxConnections: 1,
9+
rateLimit: 1000, // maxConnections will be forced to 1
10+
skipDuplicates: true,
11+
// This will be called for each crawled page
12+
callback: function (error, res, done) {
13+
if (error) {
14+
15+
// Log the error
16+
console.log(error);
17+
18+
} else {
19+
20+
// Set $ to equal res.$
21+
const $ = res.$;
22+
23+
// Write the data to a CSV file
24+
if (fs.existsSync("websites.csv")) {
25+
fs.appendFileSync("websites.csv", `\n${$("title").text().trim().replace(/,/g, "")}, ${res.request.uri.href}`);
26+
} else {
27+
fs.writeFileSync("websites.csv", `title, url\n${$("title").text().replace(/,/g, "")}, ${res.request.uri.href}`);
28+
}
29+
30+
// Add all the urls on the current website to the queue
31+
32+
// For each url on the current website
33+
$("a").each(function (i, link) {
34+
35+
try {
36+
37+
// Format the url
38+
const requestUrl = res.request.uri.href;
39+
let href = $(link).attr("href");
40+
href = URL.resolve(requestUrl, href);
41+
42+
// If the url starts with http then added to the queue
43+
if (href.startsWith("http")) {
44+
crawler.queue(href);
45+
}
46+
47+
} catch (error) {
48+
49+
}
50+
51+
});
52+
53+
}
54+
done();
55+
}
56+
});
57+
58+
// Add the starting url to the queue with default callback
59+
crawler.queue("https://github.com/bda-research/node-crawler");

‎Make your Own Web Crawler/package.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"name": "web-crawler",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "app.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"keywords": [],
10+
"author": "",
11+
"license": "MIT",
12+
"dependencies": {
13+
"crawler": "^1.2.1"
14+
}
15+
}

‎Sending SMS using Node.js/SMS Appointment with express/index.js‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// Import dependencies
2+
const express = require("express")
3+
const mongoose = require("mongoose");
4+
const MessagingResponse = require("twilio").twiml.MessagingResponse;
5+
6+
// Setup the express server
7+
const app = express();
8+
const port = 80;
9+
10+
// The mongoose appointment model
11+
let Appointment;
12+
13+
// Import middlewares into express
14+
app.use(express.json());
15+
16+
// Connect to MongoDB
17+
(async function connect() {
18+
19+
try {
20+
21+
// Try to connect to MongoDB
22+
await mongoose.connect("Connection string goes here", { useNewUrlParser: true });
23+
24+
// Log the success
25+
console.log("Mongoose connected successfully");
26+
27+
} catch (error) {
28+
29+
// Log the fail
30+
console.log("Mongoose failed to connect");
31+
32+
}
33+
34+
// Make the mongoose schema for appointments
35+
const schema = new mongoose.Schema({
36+
messageId: { type: String },
37+
accountId: { type: String },
38+
fullName: { type: String },
39+
datetime: { type: Date },
40+
message: { type: String },
41+
phoneNumber: { type: String },
42+
confirmed: {
43+
type: Boolean,
44+
default: false
45+
}
46+
});
47+
48+
// Make the mongoose Appointment model
49+
Appointment = mongoose.model("Appointment", schema);
50+
51+
})();
52+
53+
app.post("/api", async (req, res) => {
54+
55+
// The Twilio message response generator
56+
const twiml = new MessagingResponse();
57+
58+
// Wait for the Appointment to be initialized
59+
await Appointment.init();
60+
61+
if (req.body.Body.toLowerCase() == "help") {
62+
twiml.message("book full name 2019-10-29 at 11:30 custom message");
63+
} else if (req.body.Body.toLowerCase().startsWith("book")) {
64+
65+
// Regular expression
66+
const command = /book (.*?)\s(\d.*?)at\s(.*?)\s(.*)/.exec(req.body.Body);
67+
68+
// Construct the appointment
69+
const appointment = {
70+
messageId: req.body.MessageSid,
71+
accountId: req.body.AccountSid,
72+
fullName: command[1],
73+
datetime: new Date(`${command[2].trim()}T${command[3].trim()}Z`),
74+
message: command[4],
75+
phoneNumber: req.body.From,
76+
confirmed: false
77+
}
78+
79+
// Make a new appointment document and add it to the DB
80+
const appointmentDocument = new Appointment(appointment);
81+
await appointmentDocument.save();
82+
83+
// Send a message back to the user asking for confirmation
84+
twiml.message("Write yes to confirm. If you don't write yes your appointment will not be booked.");
85+
86+
} else if (req.body.Body.toLowerCase() == "yes") {
87+
88+
// Update the appointment in the DB
89+
const document = await Appointment.updateOne({ accountId: req.body.AccountSid }, { confirmed: true });
90+
91+
// Show the user if the appointment was updated
92+
if (document.nModified >= 1) {
93+
twiml.message("Your appointment is now booked.");
94+
} else {
95+
twiml.message("It doesn't look like you have an appointment to accept, maybe you already accepted your appointment.");
96+
}
97+
98+
} else {
99+
twiml.message("book full name 2019-10-29 at 11:30 custom message.");
100+
}
101+
102+
// Return the response
103+
res.writeHead(200, { "Content-Type": "text/xml" });
104+
res.end(twiml.toString());
105+
106+
});
107+
108+
// Start the server
109+
app.listen(port, () => console.log(`Messaging response listening on port ${port}!`));

‎Sending SMS using Node.js/SMS Appointment with express/package.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"name": "sms-appointment",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "index.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"keywords": [],
10+
"author": "",
11+
"license": "MIT",
12+
"dependencies": {
13+
"express": "^4.17.1",
14+
"mongoose": "^5.7.7",
15+
"twilio": "^3.36.0"
16+
}
17+
}

‎Sending SMS using Node.js/SMS Appointment with now/api/index.js‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
// Import dependencies
2+
const MessagingResponse = require("twilio").twiml.MessagingResponse;
3+
const mongoose = require("mongoose");
4+
5+
// The mongoose appointment model
6+
let Appointment;
7+
8+
// Connect to MongoDB
9+
async function connect() {
10+
11+
try {
12+
13+
// Try to connect to MongoDB
14+
await mongoose.connect("Connection string goes here", { useNewUrlParser: true });
15+
16+
// Log the success
17+
console.log("Mongoose connected successfully");
18+
19+
} catch (error) {
20+
21+
// Log the fail
22+
console.log("Mongoose failed to connect");
23+
24+
}
25+
26+
// Make the mongoose schema for appointments
27+
const schema = new mongoose.Schema({
28+
messageId: { type: String },
29+
accountId: { type: String },
30+
fullName: { type: String },
31+
datetime: { type: Date },
32+
message: { type: String },
33+
phoneNumber: { type: String },
34+
confirmed: {
35+
type: Boolean,
36+
default: false
37+
}
38+
});
39+
40+
// Make the mongoose Appointment model
41+
Appointment = mongoose.model("Appointment", schema);
42+
43+
// Wait for the Appointment to be initialized
44+
await Appointment.init();
45+
46+
}
47+
48+
module.exports = async (req, res) => {
49+
50+
// The Twilio message response generator
51+
const twiml = new MessagingResponse();
52+
53+
// Connect to MongoDB and initialize the Appointment model
54+
await connect();
55+
56+
if (req.body.Body.toLowerCase() == "help") {
57+
twiml.message("book full name 2019-10-29 at 11:30 custom message");
58+
} else if (req.body.Body.toLowerCase().startsWith("book")) {
59+
60+
// Regular expression
61+
const command = /book (.*?)\s(\d.*?)at\s(.*?)\s(.*)/.exec(req.body.Body);
62+
63+
// Construct the appointment
64+
const appointment = {
65+
messageId: req.body.MessageSid,
66+
accountId: req.body.AccountSid,
67+
fullName: command[1],
68+
datetime: new Date(`${command[2].trim()}T${command[3].trim()}Z`),
69+
message: command[4],
70+
phoneNumber: req.body.From,
71+
confirmed: false
72+
}
73+
74+
// Make a new appointment document and add it to the DB
75+
const appointmentDocument = new Appointment(appointment);
76+
await appointmentDocument.save();
77+
78+
// Send a message back to the user asking for confirmation
79+
twiml.message("Write yes to confirm. If you don't write yes your appointment will not be booked.");
80+
81+
} else if (req.body.Body.toLowerCase() == "yes") {
82+
83+
// Update the appointment in the DB
84+
const document = await Appointment.updateOne({ accountId: req.body.AccountSid }, { confirmed: true });
85+
86+
// Show the user if the appointment was updated
87+
if (document.nModified >= 1) {
88+
twiml.message("Your appointment is now booked.");
89+
} else {
90+
twiml.message("It doesn't look like you have an appointment to accept, maybe you already accepted your appointment.");
91+
}
92+
93+
} else {
94+
twiml.message("book full name 2019-10-29 at 11:30 custom message.");
95+
}
96+
97+
// Return the response
98+
res.writeHead(200, { "Content-Type": "text/xml" });
99+
res.end(twiml.toString());
100+
101+
}

‎Sending SMS using Node.js/SMS Appointment with now/now.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"name":"sms-appointment",
3+
"version":2,
4+
"builds":[
5+
{
6+
"src":"/api/*.js",
7+
"use":"@now/node"
8+
}
9+
],
10+
"routes":[
11+
{
12+
"src":"/api/(.*)",
13+
"dest":"/api/$1"
14+
}
15+
]
16+
}

‎Sending SMS using Node.js/SMS Appointment with now/package.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"name": "sms-appointment",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "api/index.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"keywords": [],
10+
"author": "",
11+
"license": "MIT",
12+
"dependencies": {
13+
"express": "^4.17.1",
14+
"mongoose": "^5.7.7",
15+
"twilio": "^3.36.0"
16+
}
17+
}

‎Sending SMS using Node.js/SMS Sender/package.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
{
2+
"name": "twilio-sms-sender",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "app.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"keywords": [],
10+
"author": "",
11+
"license": "MIT",
12+
"dependencies": {
13+
"twilio": "^3.36.0"
14+
}
15+
}

0 commit comments

Comments
 (0)