File System (node:fs & node:fs/promises)
Node.js provides three APIs to interact with the file system: Synchronous (blocks the event loop), Callback-based (asynchronous, legacy), and Promise-based (asynchronous, modern standard via node:fs/promises).
+------------------+-----------------------+-----------------------------+
| API Style | Example Method | Impact on Event Loop |
+------------------+-----------------------+-----------------------------+
| Synchronous | fs.readFileSync() | Freezes Call Stack entirely |
| Callback-based | fs.readFile() | Non-blocking (libuv thread) |
| Promise-based | fsPromises.readFile() | Non-blocking (async/await) |
+------------------+-----------------------+-----------------------------+
Production Rule: Never use synchronous file operations (
*Sync) inside request handlers or loops. Reserve them strictly for initial bootstrapping (e.g., loading a static TLS certificate or local config file before the server starts listening).
JavaScript
import fs from 'node:fs/promises';
import path from 'node:path';
const filePath = path.join(process.cwd(), 'data', 'app.log');
async function handleFileLifecycle() {
try {
// 1. Create / Overwrite (Writing)
await fs.writeFile(filePath, 'Initial system state: OK\n', { encoding: 'utf8' });
// 2. Append data
await fs.appendFile(filePath, 'Event: Worker process started\n', 'utf8');
// 3. Read data
// Without encoding, it returns a raw Buffer; with encoding, it returns a string
const contents = await fs.readFile(filePath, { encoding: 'utf8' });
console.log('File Content:\n', contents);
// 4. Update / Partial overwrite using a FileHandle
// Fine-grained control without reading the whole file into RAM
const fileHandle = await fs.open(filePath, 'r+');
try {
const updateBuffer = Buffer.from('CRITICAL');
// Write buffer at byte offset 0
await fileHandle.write(updateBuffer, 0, updateBuffer.length, 0);
} finally {
await fileHandle.close(); // Prevent file descriptor leaks
}
// 5. Check existence & metadata
const stats = await fs.stat(filePath);
console.log(`Size: ${stats.size} bytes | Is Directory: ${stats.isDirectory()}`);
// 6. Delete file
await fs.unlink(filePath);
console.log('File successfully removed');
} catch (err) {
if (err.code === 'ENOENT') {
console.error('File or directory does not exist');
} else {
console.error('FS error:', err.message);
}
}
}
Path Module (node:path)
Operating systems use different segment separators: POSIX (Linux/macOS) uses /, while Windows uses \. The node:path module abstracts these differences and provides defenses against path-based vulnerabilities.
JavaScript
import path from 'node:path';
// 1. path.join vs. path.resolve
// path.join: Concatenates pieces using the platform separator and normalizes relative segments (.. and .)
const joined = path.join('/users', 'local', '..', 'bin', 'node');
// Linux/macOS: /users/bin/node
// path.resolve: Processes arguments from right to left until an absolute path is constructed
const resolved = path.resolve('data', 'reports', '2026.pdf');
// Output: /current/working/dir/data/reports/2026.pdf
// 2. Normalization and Extraction
const dirtyPath = '/app/uploads//images/../avatar.png';
const cleanPath = path.normalize(dirtyPath); // '/app/uploads/avatar.png'
const parsed = path.parse(cleanPath);
// {
// root: '/',
// dir: '/app/uploads',
// base: 'avatar.png',
// ext: '.png',
// name: 'avatar'
// }
Defending Against Path Traversal Vulnerabilities
Malicious clients can pass inputs like ../../etc/passwd to extract system files. Use path.resolve combined with prefix validation:
JavaScript
import path from 'node:path';
const PUBLIC_DIRECTORY = path.resolve(process.cwd(), 'public');
function safeResolve(userInput) {
// 1. Resolve target file to an absolute path
const targetPath = path.resolve(PUBLIC_DIRECTORY, userInput);
// 2. Validate that targetPath strictly starts within PUBLIC_DIRECTORY
const relative = path.relative(PUBLIC_DIRECTORY, targetPath);
const isSafe = !relative.startsWith('..') && !path.isAbsolute(relative);
if (!isSafe) {
throw new Error('Access Denied: Path Traversal Attempt Detected');
}
return targetPath;
}
// safeResolve('images/cat.png') -> OK (/project/public/images/cat.png)
// safeResolve('../../../etc/shadow') -> Throws "Access Denied"
Events (node:events)
The EventEmitter class powers Node’s internal architecture (Stream, http.Server, and net.Socket all inherit from it). It implements the Observer pattern for event-driven coordination.
JavaScript
import { EventEmitter } from 'node:events';
class PaymentProcessor extends EventEmitter {
processPayment(orderId, amount) {
this.emit('payment:started', { orderId });
// Simulate payment work
setTimeout(() => {
if (amount > 0) {
this.emit('payment:success', { orderId, amount, transactionId: 'txn_987' });
} else {
// Crucial: Unhandled 'error' events crash the entire Node.js process!
this.emit('error', new Error(`Invalid amount: ${amount} for order: ${orderId}`));
}
}, 100);
}
}
const processor = new PaymentProcessor();
// Standard listener: triggers on every emission
processor.on('payment:started', ({ orderId }) => {
console.log(`[LOG] Payment init for order ${orderId}`);
});
// Single-use listener: auto-deregisters after the first call
processor.once('payment:success', (data) => {
console.log(`[ALERT] First successful payment processed:`, data);
});
// Mandatory error listener
processor.on('error', (err) => {
console.error(`[CRITICAL] Processor failed: ${err.message}`);
});
Mitigating Memory Leaks with Event Listeners
Attaching listeners inside repeated function calls without removing them will retain referenced variables in memory permanently.
JavaScript
// By default, EventEmitter warns if > 10 listeners are added to a single event
processor.setMaxListeners(20);
function onOrderComplete(data) {
console.log('Sending metrics...');
}
// Register
processor.on('order:complete', onOrderComplete);
// Cleanup: removeListener / off requires a reference to the exact named function
processor.off('order:complete', onOrderComplete);
// Alternatively: Node.js 16+ supports auto-cleaning via AbortController
const ac = new AbortController();
processor.on('data', (chunk) => console.log(chunk), { signal: ac.signal });
// Instantly unsubscribes without holding function references:
ac.abort();
Buffers & Streams
JavaScript was originally designed for strings, not raw binary bytes. Node.js introduced Buffers to allocate fixed-size memory outside the V8 heap and Streams to handle datasets larger than system memory.
Binary Data Handling with Buffer
Buffers represent chunks of raw binary data as sequences of octets (integers from 0 to 255).
JavaScript
// 1. Allocation (allocates zero-filled safe memory)
const bufZero = Buffer.alloc(10); // 10 zeroed bytes
// Unsafe allocation: faster, but retains uninitialized/dirty memory (could leak old RAM data)
// const bufUnsafe = Buffer.allocUnsafe(10);
// 2. Creation from existing data
const bufFromString = Buffer.from('Hello World', 'utf8');
console.log(bufFromString);
// <Buffer 48 65 6c 6c 6f 20 57 6f 72 6c 64> (hex representation)
// 3. Conversions
console.log(bufFromString.toString('base64')); // SGVsbG8gV29ybGQ=
console.log(bufFromString.toString('hex')); // 48656c6c6f20576f726c64
// 4. Mutation & Slicing
bufFromString[0] = 74; // ASCII for 'J'
console.log(bufFromString.toString('utf8')); // Jello World
The 4 Core Stream Types
1. Readable (fs.createReadStream, req) ──> Data Source (Producer)
2. Writable (fs.createWriteStream, res) ──> Data Sink (Consumer)
3. Duplex (net.Socket) ──> Both Readable & Writable simultaneously
4. Transform (zlib.createGzip, crypto) ──> Duplex that modifies data during pass-through
Backpressure, Piping, and stream.pipeline
Backpressure occurs when a Readable stream pushes data into a Writable stream faster than the consumer can write it (e.g., fast disk reading vs. slow network client). Without backpressure handling, unwritten chunks accumulate in RAM, triggering high memory usage and crashes.
The Anti-Pattern: readable.pipe(writable)
While pipe() handles backpressure, it does not cleanly clean up file descriptors or forward errors if one stream in the chain fails, often leading to silent stream leaks.
The Modern Pattern: stream.promises.pipeline
Use pipeline from node:stream/promises. It manages backpressure, destroys all streams if any stream encounters an error, and returns a Promise.
JavaScript
import fs from 'node:fs';
import { pipeline } from 'node:stream/promises';
import { createGzip } from 'node:zlib';
import path from 'node:path';
async function compressLargeFile(source, destination) {
try {
// Transfers 10GB+ files using only ~64KB of RAM at any given time
await pipeline(
fs.createReadStream(source, { highWaterMark: 64 * 1024 }), // 64KB chunk buffer
createGzip(), // Transform stream
fs.createWriteStream(destination) // Destination sink
);
console.log('Compression complete without RAM spikes.');
} catch (err) {
console.error('Pipeline failed:', err.message);
// All underlying file descriptors and internal stream buffers are safely closed
}
}
Chunk-by-Chunk Processing with Async Iterators
Readable streams implement the Symbol.asyncIterator contract, allowing processing of line items, JSON rows, or chunks using standard for await...of:
JavaScript
import fs from 'node:fs';
async function processLogChunks(filePath) {
const readStream = fs.createReadStream(filePath, { encoding: 'utf8' });
let totalBytes = 0;
for await (const chunk of readStream) {
totalBytes += chunk.length;
// Inspect each individual memory chunk as it arrives
console.log(`Received chunk of size ${chunk.length}`);
}
console.log(`Finished processing ${totalBytes} characters.`);
}
OS & HTTP (node:os & node:http)
The node:os Module
Used to inspect host hardware parameters, allocate thread pools, or run cluster workers.
JavaScript
import os from 'node:os';
console.log('Platform:', os.platform()); // 'linux', 'darwin', 'win32'
console.log('CPU Architecture:', os.arch()); // 'x64', 'arm64'
console.log('Available CPU Cores:', os.cpus().length);
console.log('Free RAM:', (os.freemem() / 1024 / 1024).toFixed(2), 'MB');
console.log('Total RAM:', (os.totalmem() / 1024 / 1024).toFixed(2), 'MB');
console.log('Network Interfaces:', Object.keys(os.networkInterfaces()));
Bare-Bones HTTP Server (node:http)
Under the hood, all major frameworks (Express, Fastify, NestJS) wrap node:http. In this API:
reqis a Readable Stream (http.IncomingMessage).resis a Writable Stream (http.ServerResponse).
JavaScript
import http from 'node:http';
const server = http.createServer(async (req, res) => {
const { method, url } = req;
// 1. Simple route matching
if (url === '/api/health' && method === 'GET') {
res.writeHead(200, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ status: 'healthy', uptime: process.uptime() }));
return;
}
// 2. Reading payload body from the incoming Readable Stream
if (url === '/api/echo' && method === 'POST') {
const buffers = [];
// Collect binary chunks from the network socket
for await (const chunk of req) {
buffers.push(chunk);
}
const rawBody = Buffer.concat(buffers).toString('utf8');
try {
const parsedBody = JSON.parse(rawBody || '{}');
// Writing response headers and payload
res.writeHead(200, {
'Content-Type': 'application/json',
'X-Powered-By': 'NodeJS-Native-Core'
});
res.end(JSON.stringify({ received: parsedBody, timestamp: Date.now() }));
} catch {
res.writeHead(400, { 'Content-Type': 'application/json' });
res.end(JSON.stringify({ error: 'Malformed JSON payload' }));
}
return;
}
// 3. Fallback 404 Handler
res.writeHead(404, { 'Content-Type': 'text/plain' });
res.end('Resource Not Found');
});
const PORT = 3000;
server.listen(PORT, '127.0.0.1', () => {
console.log(`HTTP Server running at http://127.0.0.1:${PORT}/`);
});
