File Uploads & Storage

What is Multer?

Multer is a Node.js middleware for handling multipart/form-data, the standard encoding used when browsers submit file uploads. It sits between Express and your route handler, parsing incoming streams and attaching file metadata to req.file (single) or req.files (array).

Memory vs Disk Storage

const multer = require('multer');
const path = require('path');

// Memory storage - file kept in RAM as Buffer
const memoryUpload = multer({
  storage: multer.memoryStorage(),
  limits: { fileSize: 5 * 1024 * 1024 } // 5MB
});

// Disk storage - file written to temp directory
const diskUpload = multer({
  storage: multer.diskStorage({
    destination: (req, file, cb) => cb(null, '/tmp/uploads'),
    filename: (req, file, cb) => {
      const unique = `${Date.now()}-${Math.round(Math.random() * 1e9)}`;
      cb(null, `${unique}${path.extname(file.originalname)}`);
    }
  }),
  limits: { fileSize: 10 * 1024 * 1024 } // 10MB
});

// Usage in Express
app.post('/upload', memoryUpload.single('avatar'), (req, res) => {
  console.log(req.file.originalname, req.file.size, req.file.buffer);
  res.json({ uploaded: true });
});

When to Use Each

• Memory storage: Small files (<5MB), quick processing needed, no disk I/O. Use for avatars, profile images, document thumbnails.
• Disk storage: Large files, or when you need to stream to S3 without buffering. Essential for videos, large PDFs, and dataset uploads.

Key Points

• Multer does not handle non-multipart forms; use express.json() for JSON payloads.
• Always set limits.fileSize to prevent denial-of-service via massive uploads.
• The fileFilter callback lets you restrict by MIME type before processing.

Why Presigned URLs?

When files are large, routing every byte through your Node.js server wastes memory and bandwidth. Presigned URLs let the client upload directly to S3 while your server controls access via a time-limited, signed URL.

Generate Presigned Upload URL

const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3');
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');

const s3 = new S3Client({ region: process.env.AWS_REGION });

app.get('/presigned-upload', async (req, res) => {
  const { filename, contentType } = req.query;

  // Validate file type
  const allowed = ['image/jpeg', 'image/png', 'application/pdf'];
  if (!allowed.includes(contentType)) {
    return res.status(400).json({ error: 'File type not allowed' });
  }

  const key = `uploads/${Date.now()}-${filename}`;
  const command = new PutObjectCommand({
    Bucket: process.env.S3_BUCKET,
    Key: key,
    ContentType: contentType,
    // Optional: server-side encryption
    ServerSideEncryption: 'AES256'
  });

  const url = await getSignedUrl(s3, command, { expiresIn: 300 });
  res.json({ uploadUrl: url, key, expiresIn: 300 });
});

// Client-side upload using fetch
async function uploadToS3(presignedUrl, file) {
  const response = await fetch(presignedUrl, {
    method: 'PUT',
    headers: { 'Content-Type': file.type },
    body: file
  });
  if (!response.ok) throw new Error('Upload failed');
  return response.headers.get('ETag');
}

Security Considerations

• Keep expiration short (5-15 minutes).
• Validate file type and size before generating the URL.
• Never expose your S3 credentials to the client.
• Store the returned ETag in your database to confirm the upload completed.

Streaming Without Buffered Memory

For files that exceed available RAM, stream the upload directly from the request to S3 using the AWS SDK v3 streaming support:

const { S3Client, PutObjectCommand } = require('@aws-sdk/client-s3');
const busboy = require('busboy');

const s3 = new S3Client({ region: process.env.AWS_REGION });

app.post('/stream-upload', (req, res) => {
  const bb = busboy({ headers: req.headers, limits: { files: 1 } });
  let uploadResult = null;

  bb.on('file', (name, file, info) => {
    const { filename, mimeType } = info;
    const key = `uploads/${Date.now()}-${filename}`;

    // Create a passthrough stream and pipe directly to S3
    const command = new PutObjectCommand({
      Bucket: process.env.S3_BUCKET,
      Key: key,
      ContentType: mimeType,
      Body: file // Node.js readable stream
    });

    s3.send(command).then((result) => {
      uploadResult = { key, etag: result.ETag };
    }).catch((err) => {
      console.error('S3 upload error:', err);
      file.resume(); // Drain the stream
    });
  });

  bb.on('finish', () => {
    res.json({ uploaded: true, ...uploadResult });
  });

  req.pipe(bb);
});

Progress Tracking with Multiple Chunks

For multi-gigabyte uploads, split the file into chunks and track progress:

// Client-side: Read file in 5MB chunks and report progress
async function uploadWithProgress(file, uploadUrl, onProgress) {
  const CHUNK_SIZE = 5 * 1024 * 1024;
  const totalChunks = Math.ceil(file.size / CHUNK_SIZE);

  for (let i = 0; i < totalChunks; i++) {
    const start = i * CHUNK_SIZE;
    const end = Math.min(start + CHUNK_SIZE, file.size);
    const chunk = file.slice(start, end);

    await fetch(uploadUrl, {
      method: 'PUT',
      body: chunk,
      headers: {
        'Content-Type': file.type,
        'Content-Range': `bytes ${start}-${end - 1}/${file.size}`
      }
    });

    onProgress({ loaded: end, total: file.size, percent: Math.round((end / file.size) * 100) });
  }
}

Key Points

• Streaming avoids OOM errors when serving thousands of concurrent uploads.
• Always call file.resume() on error to drain the stream and free memory.
• Use busboy or multer with storage: 'memory' for small files; use raw stream handling for large ones.

When to Use Multipart

S3 multipart upload is required for files over 5GB and recommended for files over 100MB. It enables parallel chunk uploads, automatic retries of failed parts, and resume-after-failure without re-uploading the entire file.

Server-Side Multipart Orchestrator

const {
  S3Client,
  CreateMultipartUploadCommand,
  UploadPartCommand,
  CompleteMultipartUploadCommand,
  AbortMultipartUploadCommand
} = require('@aws-sdk/client-s3');

const s3 = new S3Client({ region: process.env.AWS_REGION });
const CHUNK_SIZE = 5 * 1024 * 1024; // 5MB per part

async function initiateMultipartUpload(filename, contentType) {
  const key = `uploads/${Date.now()}-${filename}`;
  const command = new CreateMultipartUploadCommand({
    Bucket: process.env.S3_BUCKET,
    Key: key,
    ContentType: contentType
  });
  const result = await s3.send(command);
  return { uploadId: result.UploadId, key };
}

async function uploadPart(uploadId, key, partNumber, body) {
  const command = new UploadPartCommand({
    Bucket: process.env.S3_BUCKET,
    Key: key,
    UploadId: uploadId,
    PartNumber: partNumber,
    Body: body,
    ContentLength: body.length
  });
  const result = await s3.send(command);
  return { partNumber, ETag: result.ETag };
}

async function completeMultipartUpload(uploadId, key, parts) {
  const command = new CompleteMultipartUploadCommand({
    Bucket: process.env.S3_BUCKET,
    Key: key,
    UploadId: uploadId,
    MultipartUpload: { Parts: parts.sort((a, b) => a.partNumber - b.partNumber) }
  });
  return s3.send(command);
}

async function abortMultipartUpload(uploadId, key) {
  const command = new AbortMultipartUploadCommand({
    Bucket: process.env.S3_BUCKET,
    Key: key,
    UploadId: uploadId
  });
  return s3.send(command);
}

Client-Side Chunk Upload Flow

async function uploadLargeFile(file, uploadId, key) {
  const totalChunks = Math.ceil(file.size / CHUNK_SIZE);
  const completedParts = [];

  for (let i = 1; i <= totalChunks; i++) {
    const start = (i - 1) * CHUNK_SIZE;
    const end = Math.min(start + CHUNK_SIZE, file.size);
    const chunk = file.slice(start, end);

    const formData = new FormData();
    formData.append('chunk', chunk);
    formData.append('partNumber', i);
    formData.append('uploadId', uploadId);
    formData.append('key', key);

    const res = await fetch('/upload-part', {
      method: 'POST',
      body: formData
    });
    const { ETag } = await res.json();
    completedParts.push({ partNumber: i, ETag });
  }

  // Finalize
  await fetch('/complete-upload', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ uploadId, key, parts: completedParts })
  });
}

Key Points

• Minimum part size is 5MB (except the last part).
• Parts can be uploaded in parallel for throughput gains.
• Always handle cleanup: abort incomplete multipart uploads after a timeout to avoid storage charges.
• Track upload state in your database to support resume functionality.

Defense Layers for File Uploads

Never trust client-supplied file names or MIME types. Implement validation at multiple levels:

const { fileTypeFromFile } = require('file-type');
const sharp = require('sharp');

async function validateUpload(buffer, originalName) {
  // 1. Check MIME type from buffer (magic bytes)
  const detected = await fileTypeFromFile(buffer);
  if (!detected) throw new Error('Unrecognizable file type');

  const allowed = {
    'image/jpeg': { maxSize: 5 * 1024 * 1024 },
    'image/png': { maxSize: 5 * 1024 * 1024 },
    'application/pdf': { maxSize: 50 * 1024 * 1024 }
  };

  const rule = allowed[detected.mime];
  if (!rule) throw new Error(`File type ${detected.mime} not allowed`);
  if (buffer.length > rule.maxSize) throw new Error('File too large');

  // 2. Scan for embedded executables in PDFs
  if (detected.mime === 'application/pdf') {
    const hasJavaScript = buffer.includes(Buffer.from('/JavaScript'));
    if (hasJavaScript) throw new Error('PDF contains JavaScript');
  }

  // 3. For images, verify dimensions
  if (detected.mime.startsWith('image/')) {
    const meta = await sharp(buffer).metadata();
    if (meta.width > 4096 || meta.height > 4096) {
      throw new Error('Image dimensions exceed maximum');
    }
  }

  return { mime: detected.mime, ext: detected.ext };
}

// Middleware wrapper
const validateFileUpload = async (req, res, next) => {
  try {
    if (!req.file) return res.status(400).json({ error: 'No file provided' });
    const result = await validateUpload(req.file.buffer, req.file.originalname);
    req.validatedFile = result;
    next();
  } catch (err) {
    res.status(422).json({ error: err.message });
  }
};

Security Checklist

1. Rename files: Never use client-provided filenames. Generate UUIDs or timestamps.
2. Set Content-Type explicitly: Prevent MIME sniffing attacks.
3. Limit file size: Both at multer level and at S3 presigned URL generation.
4. Scan uploads: Use ClamAV or AWS Lambda for virus scanning on large uploads.
5. Store outside webroot: Serve files from S3, never from your static assets folder.
6. Use signed URLs for downloads: Limit who can access uploaded files.