Webhook Testing & Debugging: Complete Guide to Local Development and Troubleshooting

┌─────────────────────────────────────────────────────────────────────┐
│                    WEBHOOK TESTING COMPLEXITY                       │
├─────────────────────────────────────────────────────────────────────┤
│                                                                     │
│   Traditional API Testing          Webhook Testing                  │
│   ─────────────────────            ───────────────                  │
│   You control the request    →     Provider controls the request    │
│   Immediate response         →     Asynchronous delivery            │
│   Easy to reproduce          →     Triggered by external events     │
│   Simple auth (API key)      →     Cryptographic signatures         │
│   Test data you create       →     Real event payloads              │
│                                                                     │
│   ┌─────────┐                      ┌─────────────┐                  │
│   │ Client  │──request──▶          │  Provider   │                  │
│   │         │◀─response─           │  (Stripe,   │                  │
│   └─────────┘                      │   GitHub)   │                  │
│        │                           └─────────────┘                  │
│        │                                  │                         │
│        │                            webhook POST                    │
│        │                                  │                         │
│        │                                  ▼                         │
│        │                           ┌───────────┐                    │
│        └─── can't intercept ───────│ Your App  │                    │
│                                    └───────────┘                    │
│                                                                     │
└─────────────────────────────────────────────────────────────────────┘

# Install ngrok
npm install -g ngrok
# Or download from https://ngrok.com

# Start your local server
npm run dev  # Starts on localhost:3000

# In another terminal, create a tunnel
ngrok http 3000

Session Status                online
Account                       [email protected]
Version                       3.5.0
Region                        United States (us)
Web Interface                 http://127.0.0.1:4040
Forwarding                    https://a1b2c3d4.ngrok.io -> http://localhost:3000

Connections                   ttl     opn     rt1     rt5     p50     p90
                              0       0       0.00    0.00    0.00    0.00

// Webhook URL to configure in provider dashboard:
// https://a1b2c3d4.ngrok.io/webhooks/stripe

import express from 'express';
const app = express();

// IMPORTANT: Use raw body for signature verification
app.post('/webhooks/stripe',
  express.raw({ type: 'application/json' }),
  (req, res) => {
    console.log('Received webhook:', {
      headers: req.headers,
      body: req.body.toString()
    });

    // Process webhook...
    res.status(200).json({ received: true });
  }
);

app.listen(3000, () => {
  console.log('Server running on http://localhost:3000');
});

# Install cloudflared
brew install cloudflared
# Or download from https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/

# Create a quick tunnel (no account required)
cloudflared tunnel --url http://localhost:3000

# Install and use in one command
npx localtunnel --port 3000

# Your URL will be something like:
# https://quiet-tiger-42.loca.lt

# 1. Go to https://webhook.site
# 2. Copy your unique URL (e.g., https://webhook.site/abc-123-def)
# 3. Configure this URL in your provider's webhook settings
# 4. Trigger an event and see the payload instantly

// webhook-inspector.ts
import express from 'express';
import crypto from 'crypto';

const app = express();
const webhookLog: WebhookEntry[] = [];

interface WebhookEntry {
  id: string;
  timestamp: Date;
  method: string;
  path: string;
  headers: Record<string, string>;
  body: string;
  query: Record<string, string>;
}

// Log all incoming requests
app.use(express.raw({ type: '*/*' }), (req, res, next) => {
  const entry: WebhookEntry = {
    id: crypto.randomUUID(),
    timestamp: new Date(),
    method: req.method,
    path: req.path,
    headers: req.headers as Record<string, string>,
    body: req.body?.toString() || '',
    query: req.query as Record<string, string>
  };

  webhookLog.unshift(entry);
  if (webhookLog.length > 100) webhookLog.pop();

  console.log('\n=== Incoming Webhook ===');
  console.log('ID:', entry.id);
  console.log('Time:', entry.timestamp.toISOString());
  console.log('Path:', entry.path);
  console.log('Headers:', JSON.stringify(entry.headers, null, 2));
  console.log('Body:', entry.body);
  console.log('========================\n');

  next();
});

// Accept all webhooks
app.all('/webhooks/*', (req, res) => {
  res.status(200).json({ received: true });
});

// View logged webhooks
app.get('/inspect', (req, res) => {
  res.json(webhookLog);
});

// Get specific webhook
app.get('/inspect/:id', (req, res) => {
  const entry = webhookLog.find(e => e.id === req.params.id);
  if (entry) {
    res.json(entry);
  } else {
    res.status(404).json({ error: 'Not found' });
  }
});

// Replay a webhook to your actual handler
app.post('/replay/:id', async (req, res) => {
  const entry = webhookLog.find(e => e.id === req.params.id);
  if (!entry) {
    return res.status(404).json({ error: 'Not found' });
  }

  const targetUrl = req.query.target as string || 'http://localhost:3001/webhooks';

  try {
    const response = await fetch(targetUrl + entry.path, {
      method: 'POST',
      headers: entry.headers,
      body: entry.body
    });

    res.json({
      status: response.status,
      body: await response.text()
    });
  } catch (error) {
    res.status(500).json({ error: String(error) });
  }
});

app.listen(3000, () => {
  console.log('Webhook inspector running on http://localhost:3000');
  console.log('- Configure webhooks to: http://localhost:3000/webhooks/...');
  console.log('- View logged webhooks: http://localhost:3000/inspect');
});

# Install Stripe CLI
brew install stripe/stripe-cli/stripe

# Login to your Stripe account
stripe login

# Forward webhooks to your local server
stripe listen --forward-to localhost:3000/webhooks/stripe

# In another terminal, trigger test events
stripe trigger payment_intent.succeeded
stripe trigger customer.subscription.created
stripe trigger invoice.payment_failed

# Trigger with custom data
stripe trigger checkout.session.completed \
  --add checkout_session:metadata.order_id=12345

> Ready! Your webhook signing secret is whsec_xxxxx
> 2024-01-15 10:30:45   --> payment_intent.succeeded [evt_xxx]
> 2024-01-15 10:30:45  <-- [200] POST http://localhost:3000/webhooks/stripe

# Install GitHub CLI
brew install gh

# Forward webhooks for a specific repo
gh webhook forward \
  --repo=your-org/your-repo \
  --events=push,pull_request \
  --url=http://localhost:3000/webhooks/github

# Test with the smee.io service
# 1. Go to https://smee.io/new
# 2. Configure the smee.io URL in GitHub webhook settings
# 3. Use the smee client locally:
npx smee-client --url https://smee.io/your-channel --target http://localhost:3000/webhooks/github

// tests/webhook-signature.test.ts
import crypto from 'crypto';
import { verifyStripeSignature, verifyGitHubSignature } from '../lib/webhook-signatures';

describe('Stripe Signature Verification', () => {
  const secret = 'whsec_test_secret';

  function createStripeSignature(payload: string, timestamp: number): string {
    const signedPayload = `${timestamp}.${payload}`;
    const signature = crypto
      .createHmac('sha256', secret)
      .update(signedPayload)
      .digest('hex');
    return `t=${timestamp},v1=${signature}`;
  }

  test('accepts valid signature', () => {
    const payload = JSON.stringify({ type: 'payment_intent.succeeded' });
    const timestamp = Math.floor(Date.now() / 1000);
    const signature = createStripeSignature(payload, timestamp);

    expect(() => {
      verifyStripeSignature(payload, signature, secret);
    }).not.toThrow();
  });

  test('rejects invalid signature', () => {
    const payload = JSON.stringify({ type: 'payment_intent.succeeded' });
    const signature = 't=123456,v1=invalid_signature';

    expect(() => {
      verifyStripeSignature(payload, signature, secret);
    }).toThrow('Invalid signature');
  });

  test('rejects expired timestamp', () => {
    const payload = JSON.stringify({ type: 'payment_intent.succeeded' });
    const oldTimestamp = Math.floor(Date.now() / 1000) - 400; // 6+ minutes ago
    const signature = createStripeSignature(payload, oldTimestamp);

    expect(() => {
      verifyStripeSignature(payload, signature, secret, 300);
    }).toThrow('Timestamp too old');
  });

  test('rejects modified payload', () => {
    const originalPayload = JSON.stringify({ type: 'payment_intent.succeeded' });
    const timestamp = Math.floor(Date.now() / 1000);
    const signature = createStripeSignature(originalPayload, timestamp);

    const modifiedPayload = JSON.stringify({ type: 'payment_intent.failed' });

    expect(() => {
      verifyStripeSignature(modifiedPayload, signature, secret);
    }).toThrow('Invalid signature');
  });
});

describe('GitHub Signature Verification', () => {
  const secret = 'github_test_secret';

  function createGitHubSignature(payload: string): string {
    const signature = crypto
      .createHmac('sha256', secret)
      .update(payload)
      .digest('hex');
    return `sha256=${signature}`;
  }

  test('accepts valid SHA-256 signature', () => {
    const payload = JSON.stringify({ action: 'opened', pull_request: {} });
    const signature = createGitHubSignature(payload);

    expect(() => {
      verifyGitHubSignature(payload, signature, secret);
    }).not.toThrow();
  });
});

// tests/webhook-handlers.test.ts
import request from 'supertest';
import crypto from 'crypto';
import app from '../app';
import { db } from '../lib/database';

describe('Stripe Webhook Handler', () => {
  const webhookSecret = process.env.STRIPE_WEBHOOK_SECRET || 'test_secret';

  function createSignedRequest(payload: object) {
    const body = JSON.stringify(payload);
    const timestamp = Math.floor(Date.now() / 1000);
    const signedPayload = `${timestamp}.${body}`;
    const signature = crypto
      .createHmac('sha256', webhookSecret)
      .update(signedPayload)
      .digest('hex');

    return {
      body,
      signature: `t=${timestamp},v1=${signature}`
    };
  }

  beforeEach(async () => {
    // Clear test data
    await db.query('DELETE FROM payments WHERE id LIKE $1', ['test_%']);
    await db.query('DELETE FROM webhook_events WHERE event_id LIKE $1', ['evt_test_%']);
  });

  test('processes payment_intent.succeeded', async () => {
    const payload = {
      id: 'evt_test_123',
      type: 'payment_intent.succeeded',
      data: {
        object: {
          id: 'pi_test_456',
          amount: 2000,
          currency: 'usd',
          metadata: { order_id: 'order_test_789' }
        }
      }
    };

    const { body, signature } = createSignedRequest(payload);

    const response = await request(app)
      .post('/webhooks/stripe')
      .set('stripe-signature', signature)
      .set('content-type', 'application/json')
      .send(body);

    expect(response.status).toBe(200);
    expect(response.body).toEqual({ received: true });

    // Verify side effects
    const order = await db.query(
      'SELECT status FROM orders WHERE id = $1',
      ['order_test_789']
    );
    expect(order.rows[0]?.status).toBe('paid');
  });

  test('handles duplicate events idempotently', async () => {
    const payload = {
      id: 'evt_test_duplicate',
      type: 'payment_intent.succeeded',
      data: {
        object: {
          id: 'pi_test_dup',
          amount: 1000,
          currency: 'usd',
          metadata: { order_id: 'order_test_dup' }
        }
      }
    };

    const { body, signature } = createSignedRequest(payload);

    // Send same webhook twice
    await request(app)
      .post('/webhooks/stripe')
      .set('stripe-signature', signature)
      .set('content-type', 'application/json')
      .send(body);

    const response = await request(app)
      .post('/webhooks/stripe')
      .set('stripe-signature', signature)
      .set('content-type', 'application/json')
      .send(body);

    expect(response.status).toBe(200);
    expect(response.body).toHaveProperty('duplicate', true);

    // Should only have one record
    const events = await db.query(
      'SELECT COUNT(*) as count FROM webhook_events WHERE event_id = $1',
      ['evt_test_duplicate']
    );
    expect(events.rows[0].count).toBe('1');
  });

  test('rejects invalid signature', async () => {
    const response = await request(app)
      .post('/webhooks/stripe')
      .set('stripe-signature', 't=123,v1=invalid')
      .set('content-type', 'application/json')
      .send('{}');

    expect(response.status).toBe(401);
  });
});

// tests/fixtures/stripe-events/payment_intent.succeeded.json
{
  "id": "evt_fixture_payment_succeeded",
  "object": "event",
  "type": "payment_intent.succeeded",
  "created": 1705312800,
  "data": {
    "object": {
      "id": "pi_fixture_123",
      "object": "payment_intent",
      "amount": 5000,
      "amount_received": 5000,
      "currency": "usd",
      "status": "succeeded",
      "metadata": {
        "order_id": "order_fixture_456"
      }
    }
  }
}

// tests/webhook-fixtures.test.ts
import fs from 'fs';
import path from 'path';
import request from 'supertest';
import app from '../app';
import { createStripeSignature } from './helpers';

const FIXTURE_DIR = path.join(__dirname, 'fixtures/stripe-events');

describe('Stripe Webhook Fixtures', () => {
  const fixtures = fs.readdirSync(FIXTURE_DIR)
    .filter(f => f.endsWith('.json'))
    .map(f => ({
      name: f.replace('.json', ''),
      payload: JSON.parse(
        fs.readFileSync(path.join(FIXTURE_DIR, f), 'utf8')
      )
    }));

  test.each(fixtures)('handles $name event', async ({ payload }) => {
    const { body, signature } = createSignedRequest(payload);

    const response = await request(app)
      .post('/webhooks/stripe')
      .set('stripe-signature', signature)
      .set('content-type', 'application/json')
      .send(body);

    expect(response.status).toBe(200);
  });
});

// Debug middleware to log signature verification details
app.post('/webhooks/stripe', (req, res, next) => {
  const signature = req.headers['stripe-signature'] as string;
  const body = req.body;

  console.log('=== Signature Debug ===');
  console.log('Header:', signature);
  console.log('Body type:', typeof body);
  console.log('Body (first 200 chars):',
    typeof body === 'string' ? body.slice(0, 200) : Buffer.isBuffer(body) ? body.slice(0, 200).toString() : JSON.stringify(body).slice(0, 200)
  );

  // Parse signature header
  const parts = signature.split(',');
  for (const part of parts) {
    const [key, value] = part.split('=');
    console.log(`  ${key}: ${value}`);

    if (key === 't') {
      const timestamp = parseInt(value);
      const age = Math.floor(Date.now() / 1000) - timestamp;
      console.log(`  Timestamp age: ${age} seconds`);
    }
  }

  next();
});

// Problem 1: Body parser modifying request body
// BAD: JSON body parser runs before signature verification
app.use(express.json()); // This parses body first!
app.post('/webhooks/stripe', verifySignature, handleWebhook);

// GOOD: Use raw body for webhook endpoints
app.post('/webhooks/stripe',
  express.raw({ type: 'application/json' }),
  verifySignature,
  handleWebhook
);

// Problem 2: Wrong secret
// Check you're using the correct secret for your environment
const secret = process.env.NODE_ENV === 'production'
  ? process.env.STRIPE_WEBHOOK_SECRET_LIVE
  : process.env.STRIPE_WEBHOOK_SECRET_TEST;

// Problem 3: Clock skew
// Allow more tolerance for testing
const TOLERANCE = process.env.NODE_ENV === 'test' ? 600 : 300;
verifySignature(body, signature, secret, TOLERANCE);

// Diagnostic endpoint to verify connectivity
app.get('/webhooks/health', (req, res) => {
  res.json({
    status: 'ok',
    timestamp: new Date().toISOString(),
    headers: req.headers,
    ip: req.ip
  });
});

// Log all incoming requests to webhook paths
app.use('/webhooks', (req, res, next) => {
  console.log(`[${new Date().toISOString()}] ${req.method} ${req.path}`);
  console.log('Headers:', JSON.stringify(req.headers, null, 2));
  next();
});

// Problem: Webhook times out because processing takes too long
app.post('/webhooks/stripe', async (req, res) => {
  const event = verifySignature(req);

  // This takes 30 seconds but Stripe timeout is 20 seconds!
  await processPayment(event.data.object);
  await updateInventory(event.data.object);
  await sendNotifications(event.data.object);

  res.json({ received: true });
});

// Solution: Respond immediately, process asynchronously
import { Queue } from 'bullmq';

const webhookQueue = new Queue('webhooks');

app.post('/webhooks/stripe', async (req, res) => {
  const event = verifySignature(req);

  // Queue for async processing
  await webhookQueue.add('stripe-event', {
    eventId: event.id,
    type: event.type,
    data: event.data.object
  });

  // Respond immediately (< 1 second)
  res.json({ received: true });
});

// Implement comprehensive event logging
interface WebhookLog {
  eventId: string;
  eventType: string;
  receivedAt: Date;
  processedAt?: Date;
  status: 'received' | 'processing' | 'completed' | 'failed';
  error?: string;
}

const webhookLogger = {
  async logReceived(event: any): Promise<void> {
    await db.query(`
      INSERT INTO webhook_logs (event_id, event_type, received_at, status, payload)
      VALUES ($1, $2, NOW(), 'received', $3)
    `, [event.id, event.type, JSON.stringify(event)]);
  },

  async logProcessed(eventId: string): Promise<void> {
    await db.query(`
      UPDATE webhook_logs
      SET status = 'completed', processed_at = NOW()
      WHERE event_id = $1
    `, [eventId]);
  },

  async logFailed(eventId: string, error: string): Promise<void> {
    await db.query(`
      UPDATE webhook_logs
      SET status = 'failed', error = $2, processed_at = NOW()
      WHERE event_id = $1
    `, [eventId, error]);
  },

  async getMissing(since: Date): Promise<WebhookLog[]> {
    const result = await db.query(`
      SELECT * FROM webhook_logs
      WHERE received_at > $1
        AND status NOT IN ('completed')
      ORDER BY received_at
    `, [since]);
    return result.rows;
  }
};

// Add monitoring dashboard endpoint
app.get('/admin/webhooks/status', async (req, res) => {
  const stats = await db.query(`
    SELECT
      status,
      COUNT(*) as count,
      MIN(received_at) as oldest,
      MAX(received_at) as newest
    FROM webhook_logs
    WHERE received_at > NOW() - INTERVAL '24 hours'
    GROUP BY status
  `);

  res.json({
    last24Hours: stats.rows,
    pendingEvents: await webhookLogger.getMissing(
      new Date(Date.now() - 24 * 60 * 60 * 1000)
    )
  });
});

# .github/workflows/webhook-tests.yml
name: Webhook Integration Tests

on:
  push:
    paths:
      - 'src/webhooks/**'
      - 'tests/webhooks/**'

jobs:
  test-webhooks:
    runs-on: ubuntu-latest

    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_USER: test
          POSTGRES_PASSWORD: test
          POSTGRES_DB: test
        ports:
          - 5432:5432
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      redis:
        image: redis:7
        ports:
          - 6379:6379

    steps:
      - uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Run database migrations
        run: npm run db:migrate
        env:
          DATABASE_URL: postgres://test:test@localhost:5432/test

      - name: Run webhook tests
        run: npm run test:webhooks
        env:
          DATABASE_URL: postgres://test:test@localhost:5432/test
          REDIS_URL: redis://localhost:6379
          STRIPE_WEBHOOK_SECRET: whsec_test
          GITHUB_WEBHOOK_SECRET: gh_test

      - name: Upload test results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: webhook-test-results
          path: coverage/

# docker-compose.test.yml
version: '3.8'

services:
  app:
    build: .
    ports:
      - "3000:3000"
    environment:
      - NODE_ENV=test
      - DATABASE_URL=postgres://test:test@postgres:5432/test
      - REDIS_URL=redis://redis:6379
      - STRIPE_WEBHOOK_SECRET=whsec_test
    depends_on:
      - postgres
      - redis

  webhook-tester:
    image: node:20-alpine
    volumes:
      - ./tests:/tests
    command: >
      sh -c "npm install -g newman &&
             newman run /tests/webhook-collection.json
             --env-var base_url=http://app:3000"
    depends_on:
      - app

  postgres:
    image: postgres:15
    environment:
      POSTGRES_USER: test
      POSTGRES_PASSWORD: test
      POSTGRES_DB: test

  redis:
    image: redis:7

// lib/webhook-logger.ts
import pino from 'pino';

const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  formatters: {
    level: (label) => ({ level: label })
  }
});

export const webhookLogger = {
  received(event: any, requestId: string) {
    logger.info({
      msg: 'Webhook received',
      requestId,
      eventId: event.id,
      eventType: event.type,
      timestamp: new Date().toISOString()
    });
  },

  processing(eventId: string, requestId: string, step: string) {
    logger.info({
      msg: 'Webhook processing',
      requestId,
      eventId,
      step
    });
  },

  completed(eventId: string, requestId: string, durationMs: number) {
    logger.info({
      msg: 'Webhook completed',
      requestId,
      eventId,
      durationMs
    });
  },

  failed(eventId: string, requestId: string, error: Error) {
    logger.error({
      msg: 'Webhook failed',
      requestId,
      eventId,
      error: error.message,
      stack: error.stack
    });
  }
};

// Usage in webhook handler
app.post('/webhooks/stripe', async (req, res) => {
  const requestId = crypto.randomUUID();
  const startTime = Date.now();

  try {
    const event = verifySignature(req);
    webhookLogger.received(event, requestId);

    webhookLogger.processing(event.id, requestId, 'queueing');
    await webhookQueue.add('stripe-event', {
      event,
      requestId
    });

    webhookLogger.completed(event.id, requestId, Date.now() - startTime);
    res.json({ received: true, requestId });

  } catch (error) {
    webhookLogger.failed('unknown', requestId, error as Error);
    res.status(500).json({ error: 'Processing failed', requestId });
  }
});

// lib/webhook-metrics.ts
import { Counter, Histogram, Gauge } from 'prom-client';

export const webhookMetrics = {
  received: new Counter({
    name: 'webhook_events_received_total',
    help: 'Total webhook events received',
    labelNames: ['provider', 'event_type']
  }),

  processed: new Counter({
    name: 'webhook_events_processed_total',
    help: 'Total webhook events processed',
    labelNames: ['provider', 'event_type', 'status']
  }),

  duration: new Histogram({
    name: 'webhook_processing_duration_seconds',
    help: 'Webhook processing duration',
    labelNames: ['provider', 'event_type'],
    buckets: [0.1, 0.5, 1, 2, 5, 10]
  }),

  queueSize: new Gauge({
    name: 'webhook_queue_size',
    help: 'Current webhook queue size',
    labelNames: ['provider']
  }),

  signatureFailures: new Counter({
    name: 'webhook_signature_failures_total',
    help: 'Total signature verification failures',
    labelNames: ['provider']
  })
};

// Example Prometheus alert rules
/*
groups:
  - name: webhooks
    rules:
      - alert: WebhookProcessingErrors
        expr: rate(webhook_events_processed_total{status="failed"}[5m]) > 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: High webhook failure rate

      - alert: WebhookQueueBacklog
        expr: webhook_queue_size > 1000
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: Webhook queue backlog growing

      - alert: WebhookSignatureFailures
        expr: rate(webhook_signature_failures_total[5m]) > 1
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: Multiple signature verification failures
*/