paritybit.ca

Unnamed repository; edit this file 'description' to name the repository.
Log | Files | Refs | README | LICENSE

commit 29866db974c1bb8e354b7f964ed0c09327ff63ec
parent 784d53b14afeae3d2bb430c12cd76c4ce12c64cd
Author: Jake Bauer <jbauer@paritybit.ca>
Date:   Fri,  5 Apr 2019 18:14:31 -0400

Allow server to send large (>100M) files

The server would previously crash when serving large files due to
running out of memory because of the way that fs.readFile works. Add
a separate route which triggers when the file size is greater than 100M
in order to serve those files in manageable chunks.

Diffstat:
Mserver/app.js | 106++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 81 insertions(+), 25 deletions(-)

diff --git a/server/app.js b/server/app.js @@ -28,6 +28,18 @@ let crypto = require("crypto"); let pageHits = {}; +function read_page_hits() { + fs.readFileSync("./public/share/pageHits.json", (err, content) => { + if (err) { + console.log("No existing page hit data."); + } + else { + console.log("Reading in existing page hit data."); + pageHits = JSON.parse(content); + } + }); +} + function save_page_hits() { fs.writeFile("./public/share/pageHits.json", JSON.stringify(pageHits), err => { if (err) { @@ -61,6 +73,59 @@ function compute_etag(content) { return hash.read(); } +function serve_regular_file(filePath, contentType, req, res) { + fs.readFile(filePath, (err, content) => { + if (err) { + handle_error(err, res); + } + else { + let etag = compute_etag(content); + // If the resource is cached, send code 304 and don't send resource + if (etag === req.headers["if-none-match"]) { + res.writeHead(304, {"Content-Type": contentType, + "Strict-Transport-Security": "max-age=604800;" + + " includeSubDomains", "Cache-Control": "max-age=120", + "ETag": etag }); + res.end(); + } + else { + // Otherwise, send the file + res.writeHead(200, {"Content-Type": contentType, + "Strict-Transport-Security": "max-age=604800;" + +" includeSubDomains", "Cache-Control": "max-age=120", + "ETag": etag}); + res.end(content, "utf-8"); + } + } + }); +} + +function serve_large_file(filePath, fileSize, contentType, req, res) { + // The size of the file is sent as the ETag to prevent the server from + // having to do expensive hashing of large files. + let etag = fileSize; + if (etag === req.headers["if-none-match"]) { + res.writeHead(304, {"Content-Type": contentType, + "Content-Length": fileSize, + "Strict-Transport-Security": "max-age=604800; includeSubDomains", + "ETag": etag, + "Cache-Control": "max-age=120"}); + res.end(); + } + else { + // Otherwise, send the file + res.writeHead(200, {"Content-Type": contentType, + "Content-Length": fileSize, + "Strict-Transport-Security": "max-age=604800; includeSubDomains", + "ETag": etag, + "Cache-Control": "max-age=120"}); + let readStream = fs.createReadStream(filePath); + readStream.on("open", () => { + readStream.pipe(res); + }); + } +} + const httpServer = http.createServer((req, res) => { let mimeTypes = { '.html': 'text/html', @@ -77,17 +142,16 @@ const httpServer = http.createServer((req, res) => { '.mp4': 'video/mp4', '.mkv': 'video/x-matroska' } - let extName, contentType, acceptEncoding; - let filePath = "./public" + req.url; + let filePath = "./public" + req.url; // Serve the default page if none specified if (req.url === "/") { req.url = "/home.html"; } // Get the extension of the file requested and therefore the content type - extName = String(path.extname(req.url)).toLowerCase(); - contentType = mimeTypes[extName] || "application/octet-stream"; + let extName = String(path.extname(req.url)).toLowerCase(); + let contentType = mimeTypes[extName] || "application/octet-stream"; // Append html directory for serving pages (other resources will be // addressed directly as (e.g.) "/css/base.min.css") @@ -95,37 +159,29 @@ const httpServer = http.createServer((req, res) => { filePath = "./public/html" + req.url; } - fs.readFile(filePath, (err, content) => { + // Count page hits + if (pageHits[filePath] >= 1) { + pageHits[filePath] += 1; + } + else { + pageHits[filePath] = 1; + } + + fs.stat(filePath, (err, stat) => { if (err) { handle_error(err, res); } else { - if (pageHits[filePath] >= 1) { - pageHits[filePath] += 1; + if (stat.size > 100000) { + serve_large_file(filePath, stat.size, contentType, req, res); } else { - pageHits[filePath] = 1; - } - let etag = compute_etag(content); - // If the resource is cached, send code 304 and don't send resource - if (etag === req.headers["if-none-match"]) { - res.writeHead(304, {"Content-Type": contentType, - "Strict-Transport-Security": "max-age=604800;" - + " includeSubDomains", "Cache-Control": "max-age=120", - "ETag": etag }); - res.end(); - } - else { - // Otherwise, send the file - res.writeHead(200, {"Content-Type": contentType, - "Strict-Transport-Security": "max-age=604800;" - +" includeSubDomains", "Cache-Control": "max-age=120", - "ETag": etag}); - res.end(content, "utf-8"); + serve_regular_file(filePath, contentType, req, res); } } }); }).listen(8080); +read_page_hits(); setInterval(save_page_hits, 2000); console.log("Server running.");