paritybit.ca

commit 29866db974c1bb8e354b7f964ed0c09327ff63ec
parent 784d53b14afeae3d2bb430c12cd76c4ce12c64cd
Author: Jake Bauer <jbauer@paritybit.ca>
Date:   Fri,  5 Apr 2019 18:14:31 -0400

Allow server to send large (>100M) files

The server would previously crash when serving large files due to
running out of memory because of the way that fs.readFile works. Add
a separate route which triggers when the file size is greater than 100M
in order to serve those files in manageable chunks.

Diffstat:
M
server/app.js
 | 
106
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------

1 file changed, 81 insertions(+), 25 deletions(-)
diff --git a/server/app.js b/server/app.js
@@ -28,6 +28,18 @@ let crypto = require("crypto");
 
 let pageHits = {};
 
+function read_page_hits() {
+    fs.readFileSync("./public/share/pageHits.json", (err, content) => {
+        if (err) {
+            console.log("No existing page hit data.");
+        }
+        else {
+            console.log("Reading in existing page hit data.");
+            pageHits = JSON.parse(content);
+        }
+    });
+}
+
 function save_page_hits() {
     fs.writeFile("./public/share/pageHits.json", JSON.stringify(pageHits), err => {
         if (err) {
@@ -61,6 +73,59 @@ function compute_etag(content) {
     return hash.read();
 }
 
+function serve_regular_file(filePath, contentType, req, res) {
+    fs.readFile(filePath, (err, content) => {
+        if (err) {
+            handle_error(err, res);
+        }
+        else {
+            let etag = compute_etag(content);
+            // If the resource is cached, send code 304 and don't send resource
+            if (etag === req.headers["if-none-match"]) {
+                res.writeHead(304, {"Content-Type": contentType,
+                    "Strict-Transport-Security": "max-age=604800;"
+                    + " includeSubDomains", "Cache-Control": "max-age=120",
+                    "ETag": etag });
+                res.end();
+            }
+            else {
+                // Otherwise, send the file
+                res.writeHead(200, {"Content-Type": contentType,
+                    "Strict-Transport-Security": "max-age=604800;"
+                    +" includeSubDomains", "Cache-Control": "max-age=120",
+                    "ETag": etag});
+                res.end(content, "utf-8");
+            }
+        }
+    });
+}
+
+function serve_large_file(filePath, fileSize, contentType, req, res) {
+    // The size of the file is sent as the ETag to prevent the server from
+    // having to do expensive hashing of large files.
+    let etag = fileSize;
+    if (etag === req.headers["if-none-match"]) {
+        res.writeHead(304, {"Content-Type": contentType,
+            "Content-Length": fileSize,
+            "Strict-Transport-Security": "max-age=604800; includeSubDomains",
+            "ETag": etag,
+            "Cache-Control": "max-age=120"});
+        res.end();
+    }
+    else {
+        // Otherwise, send the file
+        res.writeHead(200, {"Content-Type": contentType,
+            "Content-Length": fileSize,
+            "Strict-Transport-Security": "max-age=604800; includeSubDomains",
+            "ETag": etag,
+            "Cache-Control": "max-age=120"});
+        let readStream = fs.createReadStream(filePath);
+        readStream.on("open", () => {
+            readStream.pipe(res);
+        });
+    }
+}
+
 const httpServer = http.createServer((req, res) => {
     let mimeTypes = {
         '.html': 'text/html',
@@ -77,17 +142,16 @@ const httpServer = http.createServer((req, res) => {
         '.mp4': 'video/mp4',
         '.mkv': 'video/x-matroska'
     }
-    let extName, contentType, acceptEncoding;
-    let filePath = "./public" + req.url;
 
+    let filePath = "./public" + req.url;
     // Serve the default page if none specified
     if (req.url === "/") {
         req.url = "/home.html";
     }
 
     // Get the extension of the file requested and therefore the content type
-    extName = String(path.extname(req.url)).toLowerCase();
-    contentType = mimeTypes[extName] || "application/octet-stream";
+    let extName = String(path.extname(req.url)).toLowerCase();
+    let contentType = mimeTypes[extName] || "application/octet-stream";
 
     // Append html directory for serving pages (other resources will be
     // addressed directly as (e.g.) "/css/base.min.css")
@@ -95,37 +159,29 @@ const httpServer = http.createServer((req, res) => {
         filePath = "./public/html" + req.url;
     }
 
-    fs.readFile(filePath, (err, content) => {
+    // Count page hits
+    if (pageHits[filePath] >= 1) {
+        pageHits[filePath] += 1;
+    }
+    else {
+        pageHits[filePath] = 1;
+    }
+
+    fs.stat(filePath, (err, stat) => {
         if (err) {
             handle_error(err, res);
         }
         else {
-            if (pageHits[filePath] >= 1) {
-                pageHits[filePath] += 1;
+            if (stat.size > 100000) {
+                serve_large_file(filePath, stat.size, contentType, req, res);
             }
             else {
-                pageHits[filePath] = 1;
-            }
-            let etag = compute_etag(content);
-            // If the resource is cached, send code 304 and don't send resource
-            if (etag === req.headers["if-none-match"]) {
-                res.writeHead(304, {"Content-Type": contentType,
-                    "Strict-Transport-Security": "max-age=604800;"
-                    + " includeSubDomains", "Cache-Control": "max-age=120",
-                    "ETag": etag });
-                res.end();
-            }
-            else {
-                // Otherwise, send the file
-                res.writeHead(200, {"Content-Type": contentType,
-                    "Strict-Transport-Security": "max-age=604800;"
-                    +" includeSubDomains", "Cache-Control": "max-age=120",
-                    "ETag": etag});
-                res.end(content, "utf-8");
+                serve_regular_file(filePath, contentType, req, res);
             }
         }
     });
 }).listen(8080);
 
+read_page_hits();
 setInterval(save_page_hits, 2000);
 console.log("Server running.");