Dead Letter Queue(DLQ)

Definition:

Dead Letter Queue (DLQ) is a service-side queue that stores messages that couldn’t be processed successfully after multiple retry attempts. When a message fails repeatedly due to errors, corruption, or processing issues, it gets moved to the DLQ instead of being lost or continuously retried, allowing developers to investigate and handle problematic messages separately.

Key Properties:

• Failure isolation: Separates problematic messages from healthy flow
• Retry exhaustion: Triggered after max retry attempts reached
• Message preservation: Keeps failed messages for analysis
• System stability: Prevents infinite retry loops
• Manual intervention: Allows debugging and reprocessing

How It Works:

• Message fails processing in main queue
• Retry mechanism attempts reprocessing (1-N times)
• Still failing? Message moved to DLQ
• Continue processing other messages normally
• Investigate DLQ messages separately for root cause analysis and resolution

Example:

Email Service

class EmailService {
  constructor() {
    this.emailQueue = new Queue('emails');
    this.emailDLQ = new Queue('emails-dlq');
    this.maxRetries = 5;
  }

  async sendEmailWithDLQ(emailData) {
    const message = {
      id: generateUniqueId(),
      body: emailData,
      retryCount: 0,
      createdAt: Date.now()
    };

    await this.emailQueue.send(message);
  }

  async processEmailQueue() {
    while (true) {
      const message = await this.emailQueue.receive();

      if (message) {
        await this.handleEmailMessage(message);
      }

      await this.sleep(1000);
    }
  }

  async handleEmailMessage(message) {
    try {
      const email = message.body;

      // Validate email
      if (!this.isValidEmail(email)) {
        throw new Error('Invalid email format');
      }

      // Send email
      await this.sendEmail(email);

      console.log(`📧 Email sent to ${email.to}`);

    } catch (error) {
      const retryCount = message.retryCount || 0;

      if (retryCount >= this.maxRetries) {
        // Move to DLQ with error details
        await this.emailDLQ.send({
          originalEmail: message.body,
          errorType: error.name,
          errorMessage: error.message,
          retryCount: retryCount,
          failedAt: new Date().toISOString(),

          // Add context for debugging
          serverResponse: error.response,
          emailProvider: 'sendgrid',
          lastAttempt: Date.now()
        });

        console.log(`💀 Email moved to DLQ: ${email.to}`);

        // Alert ops team
        await this.alertOpsTeam({
          message: 'Email moved to Dead Letter Queue',
          recipient: message.body.to,
          error: error.message
        });

      } else {
        // Retry with exponential backoff
        message.retryCount = retryCount + 1;
        message.retryAt = Date.now() + (1000 * Math.pow(2, retryCount));

        setTimeout(async () => {
          await this.emailQueue.send(message);
        }, 1000 * Math.pow(2, retryCount));

        console.log(`🔄 Email retry ${retryCount + 1}/${this.maxRetries}: ${message.body.to}`);
      }
    }
  }

  // DLQ Management Dashboard
  async getDLQStats() {
    const dlqMessages = await this.emailDLQ.getAllMessages();

    const stats = {
      totalFailed: dlqMessages.length,
      failureTypes: {},
      oldestFailure: null,
      recentFailures: 0
    };

    const now = Date.now();
    const oneHourAgo = now - (60 * 60 * 1000);

    dlqMessages.forEach(msg => {
      // Count failure types
      const errorType = msg.errorType || 'Unknown';
      stats.failureTypes[errorType] = (stats.failureTypes[errorType] || 0) + 1;

      // Find oldest failure
      if (!stats.oldestFailure || msg.failedAt < stats.oldestFailure) {
        stats.oldestFailure = msg.failedAt;
      }

      // Count recent failures
      if (msg.lastAttempt > oneHourAgo) {
        stats.recentFailures++;
      }
    });

    return stats;
  }

  async reprocessDLQMessages() {
    const dlqMessages = await this.emailDLQ.getAllMessages();
    let reprocessed = 0;

    for (const msg of dlqMessages) {
      try {
        // Attempt to fix common issues
        const fixedEmail = await this.fixEmailIssues(msg.originalEmail);

        if (fixedEmail) {
          // Send back to main queue for reprocessing
          await this.sendEmailWithDLQ(fixedEmail);

          // Remove from DLQ
          await this.emailDLQ.delete(msg);

          reprocessed++;
          console.log(`🔧 Reprocessed email: ${fixedEmail.to}`);
        }
      } catch (error) {
        console.log(`❌ Cannot fix email: ${error.message}`);
      }
    }

    return { reprocessed, remaining: dlqMessages.length - reprocessed };
  }
}

// Usage
const emailService = new EmailService();

// Start processing
emailService.processEmailQueue();

// Monitor DLQ (run periodically)
setInterval(async () => {
  const stats = await emailService.getDLQStats();
  console.log('DLQ Stats:', stats);

  if (stats.totalFailed > 10) {
    console.log('🚨 High number of failed emails in DLQ!');
  }
}, 60000); // Check every minute