Scaling PDF Generation at Enterprise Level: From 1K to 1M PDFs Per Month 📈

// ❌ This breaks at scale
app.post('/generate-invoice', async (req, res) => {
  const pdf = await generatePDF(req.body); // Blocks for 2-5 seconds
  res.json({ pdf_url: pdf.url });
  // Can't handle > 10 concurrent requests
});

import Bull from 'bull';
 
const pdfQueue = new Bull('pdf-generation', {
  redis: process.env.REDIS_URL,
  settings: {
    maxStalledCount: 3,
    stalledInterval: 30000
  }
});
 
// Producer: Accept requests instantly
app.post('/generate-invoice', async (req, res) => {
  const job = await pdfQueue.add('generate-pdf', {
    template: 'invoice',
    data: req.body,
    user_id: req.user.id
  }, {
    attempts: 3,
    backoff: {
      type: 'exponential',
      delay: 2000
    }
  });
 
  res.json({
    job_id: job.id,
    status: 'queued',
    status_url: `/pdf-status/${job.id}`
  });
  // Returns in < 50ms
});
 
// Consumer: Process jobs in background
pdfQueue.process('generate-pdf', 10, async (job) => {
  const { template, data, user_id } = job.data;
 
  // Update progress
  job.progress(10);
 
  const pdf = await generatePDF(template, data);
 
  job.progress(80);
 
  const url = await uploadToS3(pdf);
 
  job.progress(100);
 
  return { pdf_url: url };
});
 
// Handle completion
pdfQueue.on('completed', async (job, result) => {
  await db.pdfs.update(
    { job_id: job.id },
    { status: 'completed', pdf_url: result.pdf_url }
  );
 
  await notifyUser(job.data.user_id, result.pdf_url);
});
 
// Handle failures
pdfQueue.on('failed', async (job, err) => {
  await db.pdfs.update(
    { job_id: job.id },
    { status: 'failed', error: err.message }
  );
 
  await alerting.notify({
    severity: 'error',
    message: `PDF generation failed for job ${job.id}`,
    error: err.message
  });
});

# docker-compose.yml
version: '3.8'
 
services:
  api:
    image: your-app:latest
    deploy:
      replicas: 3
    environment:
      - REDIS_URL=redis://redis:6379
      - DATABASE_URL=postgresql://db:5432/app
 
  pdf-worker:
    image: your-pdf-worker:latest
    deploy:
      replicas: 10  # Scale workers independently
    environment:
      - REDIS_URL=redis://redis:6379
      - CONCURRENCY=5  # Each worker processes 5 jobs concurrently
 
  redis:
    image: redis:7-alpine
    volumes:
      - redis-data:/data
 
  postgres:
    image: postgres:15
    volumes:
      - postgres-data:/var/lib/postgresql/data

// worker.js
const CONCURRENCY = parseInt(process.env.CONCURRENCY || '5');
 
pdfQueue.process('generate-pdf', CONCURRENCY, async (job) => {
  // Process job
});
 
// Graceful shutdown
process.on('SIGTERM', async () => {
  console.log('SIGTERM received, shutting down gracefully...');
 
  await pdfQueue.close();
  process.exit(0);
});

const queues = {
  high: new Bull('pdf-high-priority', { redis: redisConfig }),
  normal: new Bull('pdf-normal-priority', { redis: redisConfig }),
  low: new Bull('pdf-low-priority', { redis: redisConfig })
};
 
function getPriority(jobType, userTier) {
  // Paying customers get priority
  if (userTier === 'enterprise') return 'high';
 
  // Real-time invoices are high priority
  if (jobType === 'invoice') return 'high';
 
  // Bulk reports are low priority
  if (jobType === 'bulk-report') return 'low';
 
  return 'normal';
}
 
app.post('/generate-pdf', async (req, res) => {
  const priority = getPriority(req.body.type, req.user.tier);
 
  const job = await queues[priority].add({
    template: req.body.template,
    data: req.body.data
  });
 
  res.json({ job_id: job.id, priority });
});
 
// Allocate workers by priority
queues.high.process(20, processJob);   // 20 workers for high priority
queues.normal.process(10, processJob); // 10 workers for normal
queues.low.process(5, processJob);     // 5 workers for low (use spare capacity)

// ❌ N+1 query problem
for (const job of jobs) {
  const user = await db.users.findOne({ id: job.user_id });
  const template = await db.templates.findOne({ id: job.template_id });
  // 1,000 jobs = 2,000 queries
}

// ✅ Batch load
const userIds = jobs.map(j => j.user_id);
const templateIds = jobs.map(j => j.template_id);
 
const [users, templates] = await Promise.all([
  db.users.find({ id: { $in: userIds } }),
  db.templates.find({ id: { $in: templateIds } })
]);
 
const userMap = new Map(users.map(u => [u.id, u]));
const templateMap = new Map(templates.map(t => [t.id, t]));
 
// 1,000 jobs = 2 queries

// Database connection pool
const pool = new Pool({
  host: process.env.DB_HOST,
  database: process.env.DB_NAME,
  max: 20,              // Max 20 connections
  idleTimeoutMillis: 30000,
  connectionTimeoutMillis: 2000,
});
 
// Use pool instead of creating new connections
const result = await pool.query('SELECT * FROM pdfs WHERE id = $1', [id]);

// Write to primary
const writePrimary = new Pool({ host: 'primary.db.example.com' });
 
// Read from replicas (load balanced)
const readReplicas = [
  new Pool({ host: 'replica1.db.example.com' }),
  new Pool({ host: 'replica2.db.example.com' }),
  new Pool({ host: 'replica3.db.example.com' })
];
 
function getReadPool() {
  return readReplicas[Math.floor(Math.random() * readReplicas.length)];
}
 
// Write operations
await writePrimary.query('INSERT INTO pdfs ...');
 
// Read operations (distributed across replicas)
await getReadPool().query('SELECT * FROM pdfs WHERE user_id = $1', [userId]);

import NodeCache from 'node-cache';
import Redis from 'ioredis';
 
// Layer 1: In-memory (fastest, limited size)
const memoryCache = new NodeCache({
  stdTTL: 300,      // 5 minutes
  maxKeys: 1000     // Keep only 1,000 PDFs in memory
});
 
// Layer 2: Redis (fast, distributed)
const redis = new Redis(process.env.REDIS_URL);
 
// Layer 3: S3 (slow, unlimited)
// PDFs always stored here
 
async function getCachedPDF(key) {
  // Try memory cache
  let pdf = memoryCache.get(key);
  if (pdf) {
    metrics.counter('cache.memory.hit');
    return pdf;
  }
 
  // Try Redis cache
  pdf = await redis.getBuffer(key);
  if (pdf) {
    metrics.counter('cache.redis.hit');
    memoryCache.set(key, pdf); // Promote to memory
    return pdf;
  }
 
  // Try S3
  metrics.counter('cache.miss');
  pdf = await downloadFromS3(key);
 
  if (pdf) {
    // Cache in both layers
    redis.setex(key, 3600, pdf);  // 1 hour in Redis
    memoryCache.set(key, pdf);     // 5 minutes in memory
  }
 
  return pdf;
}

// Pre-warm cache for predictable traffic
async function warmCache() {
  // Top 100 most-requested PDFs
  const popularPDFs = await db.pdfs.find()
    .sort({ request_count: -1 })
    .limit(100);
 
  for (const pdf of popularPDFs) {
    const pdfBuffer = await downloadFromS3(pdf.s3_key);
    await redis.setex(pdf.id, 3600, pdfBuffer);
  }
 
  console.log(`Warmed cache with ${popularPDFs.length} PDFs`);
}
 
// Run on startup and periodically
warmCache();
setInterval(warmCache, 60 * 60 * 1000); // Every hour

# k8s/pdf-worker-hpa.yml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: pdf-worker
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: pdf-worker
  minReplicas: 5
  maxReplicas: 50
  metrics:
  - type: External
    external:
      metric:
        name: redis_queue_depth
        selector:
          matchLabels:
            queue: pdf-generation
      target:
        type: AverageValue
        averageValue: "100"  # Scale when queue > 100 jobs per pod
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60
      policies:
      - type: Percent
        value: 100  # Double pods if needed
        periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
      - type: Percent
        value: 50   # Scale down slowly
        periodSeconds: 60

// Expose queue metrics for autoscaler
app.get('/metrics', async (req, res) => {
  const waiting = await pdfQueue.getWaitingCount();
  const active = await pdfQueue.getActiveCount();
  const failed = await pdfQueue.getFailedCount();
 
  res.set('Content-Type', 'text/plain');
  res.send(`
# HELP pdf_queue_waiting Number of jobs waiting in queue
# TYPE pdf_queue_waiting gauge
pdf_queue_waiting ${waiting}
 
# HELP pdf_queue_active Number of jobs actively processing
# TYPE pdf_queue_active gauge
pdf_queue_active ${active}
 
# HELP pdf_queue_failed Number of failed jobs
# TYPE pdf_queue_failed gauge
pdf_queue_failed ${failed}
  `);
});

import rateLimit from 'express-rate-limit';
 
// Per-user rate limit
const userLimiter = rateLimit({
  windowMs: 60 * 1000,  // 1 minute
  max: async (req) => {
    // Different limits by tier
    const limits = {
      free: 10,
      pro: 100,
      enterprise: 1000
    };
    return limits[req.user.tier] || limits.free;
  },
  keyGenerator: (req) => req.user.id,
  message: 'Too many PDF generation requests. Please try again later.'
});
 
app.post('/generate-pdf', userLimiter, async (req, res) => {
  // Only accept if queue isn't overwhelmed
  const queueDepth = await pdfQueue.getWaitingCount();
 
  if (queueDepth > 10000) {
    return res.status(503).json({
      error: 'Service temporarily unavailable. Queue is full.',
      retry_after: 60
    });
  }
 
  // Accept job
  const job = await pdfQueue.add(req.body);
  res.json({ job_id: job.id });
});

import { metrics } from './monitoring';
 
async function processJob(job) {
  const startTime = Date.now();
 
  try {
    const pdf = await generatePDF(job.data);
 
    // Track success metrics
    metrics.histogram('pdf.generation.duration', Date.now() - startTime, {
      template: job.data.template,
      user_tier: job.data.user_tier
    });
 
    metrics.counter('pdf.generation.success', 1, {
      template: job.data.template
    });
 
    metrics.histogram('pdf.file.size', pdf.length, {
      template: job.data.template
    });
 
    return pdf;
 
  } catch (err) {
    // Track failure metrics
    metrics.counter('pdf.generation.failure', 1, {
      template: job.data.template,
      error_type: err.name
    });
 
    throw err;
  }
}
 
// Alert on anomalies
setInterval(async () => {
  const queueDepth = await pdfQueue.getWaitingCount();
  const oldestJob = await pdfQueue.getJob(await pdfQueue.getNextJobId());
 
  if (queueDepth > 5000) {
    await alerting.notify({
      severity: 'warning',
      message: `Queue depth is ${queueDepth}`
    });
  }
 
  if (oldestJob && Date.now() - oldestJob.timestamp > 300000) {
    await alerting.notify({
      severity: 'critical',
      message: 'Jobs waiting over 5 minutes in queue'
    });
  }
}, 60000); // Check every minute

// Automatically delete old PDFs
const s3LifecyclePolicy = {
  Rules: [
    {
      Id: 'DeleteOldInvoices',
      Status: 'Enabled',
      Expiration: { Days: 90 },
      Filter: { Prefix: 'invoices/' }
    },
    {
      Id: 'ArchiveOldReports',
      Status: 'Enabled',
      Transitions: [
        {
          Days: 30,
          StorageClass: 'GLACIER_IR'  // Cheaper storage after 30 days
        }
      ],
      Filter: { Prefix: 'reports/' }
    }
  ]
};

# k8s/pdf-worker-deployment.yml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: pdf-worker
spec:
  replicas: 10
  template:
    spec:
      nodeSelector:
        node-type: spot  # Use spot instances (70% cheaper)
      tolerations:
      - key: spot
        operator: Equal
        value: "true"
        effect: NoSchedule