Eve library

Webhooks Integration Guide

Guide

Revision 1

On this page

This comprehensive guide walks you through implementing webhook receivers for the Lilith API, from basic setup to production-ready implementations with proper security and error handling.

Table of Contents#

Overview#

Webhooks enable your application to receive real-time notifications when events occur in the Lilith platform. Instead of polling the API for updates, webhooks push data to your server as events happen.

When to Use Webhooks#

Use Case Recommended Approach
Real-time updates ✅ Webhooks
Occasional status checks ❌ API polling
Background processing ✅ Webhooks
User-initiated actions ❌ Direct API calls
Payment confirmations ✅ Webhooks
Content publishing notifications ✅ Webhooks

Event Types#

The Lilith API supports the following webhook event categories:

Content Events:

  • content.created - New content published
  • content.updated - Content modified
  • content.deleted - Content removed

Entity Events:

  • entity.created - New entity added
  • entity.updated - Entity modified
  • entity.deleted - Entity removed

System Events:

  • system.status.changed - System status update
  • system.maintenance.scheduled - Maintenance notification
  • system.error - System error occurred

User Events:

  • user.subscription.created - New subscription
  • user.subscription.updated - Subscription changed
  • user.subscription.cancelled - Subscription ended

Payment Events:

  • payment.succeeded - Payment completed
  • payment.failed - Payment failed
  • payment.refunded - Payment refunded

Quick Start#

1. Create a Webhook Endpoint#

First, create an endpoint on your server to receive webhook events:

javascript
// Node.js with Express
const express = require('express');
const crypto = require('crypto');

const app = express();

// IMPORTANT: Use raw body for signature verification
app.post(
  '/webhooks/lilith',
  express.raw({ type: 'application/json' }),
  (req, res) => {
    const signature = req.headers['x-lilith-signature'];
    const timestamp = req.headers['x-lilith-timestamp'];

    // Verify signature (implementation below)
    if (!verifySignature(req.body, signature, timestamp)) {
      return res.status(401).json({ error: 'Invalid signature' });
    }

    const event = JSON.parse(req.body);
    console.log('Received webhook:', event.type);

    // Acknowledge receipt immediately
    res.status(200).json({ received: true });

    // Process event asynchronously
    processWebhookEvent(event);
  }
);

app.listen(3000);

2. Register Your Webhook#

bash
curl -X POST "https://api.lilith.app/v1/webhooks" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://your-app.com/webhooks/lilith",
    "events": ["content.created", "content.updated", "payment.succeeded"],
    "description": "Production webhook endpoint"
  }'

3. Handle Events#

javascript
async function processWebhookEvent(event) {
  switch (event.type) {
    case 'content.created':
      await handleNewContent(event.data);
      break;
    case 'content.updated':
      await handleContentUpdate(event.data);
      break;
    case 'payment.succeeded':
      await handlePaymentSuccess(event.data);
      break;
    default:
      console.log('Unhandled event type:', event.type);
  }
}

Setting Up Your Webhook Endpoint#

Requirements#

Your webhook endpoint must meet these requirements:

Requirement Description
HTTPS TLS 1.2+ required (no HTTP)
Response Time Respond within 30 seconds
Status Code Return 2xx for success
Availability Should be highly available
Idempotent Handle duplicate events

Node.js / Express Implementation#

javascript
const express = require('express');
const crypto = require('crypto');

const app = express();
const WEBHOOK_SECRET = process.env.LILITH_WEBHOOK_SECRET;

// Raw body middleware for signature verification
const rawBodyMiddleware = express.raw({
  type: 'application/json',
  limit: '5mb', // Adjust based on expected payload size
});

// Webhook endpoint
app.post('/webhooks/lilith', rawBodyMiddleware, async (req, res) => {
  const signature = req.headers['x-lilith-signature'];
  const timestamp = req.headers['x-lilith-timestamp'];
  const eventId = req.headers['x-lilith-event-id'];

  // 1. Verify timestamp freshness (prevent replay attacks)
  const currentTime = Math.floor(Date.now() / 1000);
  const webhookTime = parseInt(timestamp, 10);

  if (Math.abs(currentTime - webhookTime) > 300) {
    // 5 minute tolerance
    console.error('Webhook timestamp too old:', { eventId, webhookTime });
    return res.status(401).json({ error: 'Timestamp expired' });
  }

  // 2. Verify signature
  if (!verifySignature(req.body, signature, timestamp, WEBHOOK_SECRET)) {
    console.error('Invalid webhook signature:', { eventId });
    return res.status(401).json({ error: 'Invalid signature' });
  }

  // 3. Check for duplicate (idempotency)
  const isDuplicate = await checkDuplicateEvent(eventId);
  if (isDuplicate) {
    console.log('Duplicate event received:', eventId);
    return res.status(200).json({ received: true, duplicate: true });
  }

  // 4. Acknowledge immediately
  res.status(200).json({ received: true });

  // 5. Process asynchronously
  const event = JSON.parse(req.body);
  processEventAsync(event, eventId).catch((err) => {
    console.error('Event processing failed:', { eventId, error: err.message });
  });
});

function verifySignature(payload, signature, timestamp, secret) {
  const signedPayload = `${timestamp}.${payload}`;
  const expectedSignature = crypto
    .createHmac('sha256', secret)
    .update(signedPayload)
    .digest('hex');

  return crypto.timingSafeEqual(
    Buffer.from(signature),
    Buffer.from(expectedSignature)
  );
}

async function checkDuplicateEvent(eventId) {
  // Check your database/cache for this event ID
  // Return true if already processed
  const exists = await redis.exists(`webhook:${eventId}`);
  if (!exists) {
    // Mark as processing with TTL
    await redis.setex(`webhook:${eventId}`, 86400, 'processing');
  }
  return exists;
}

async function processEventAsync(event, eventId) {
  try {
    await processWebhookEvent(event);
    await redis.set(`webhook:${eventId}`, 'completed');
  } catch (error) {
    await redis.set(`webhook:${eventId}`, 'failed');
    throw error;
  }
}

Python / Flask Implementation#

python
from flask import Flask, request, jsonify
import hmac
import hashlib
import time
import json
import asyncio
from functools import wraps

app = Flask(__name__)
WEBHOOK_SECRET = os.environ.get('LILITH_WEBHOOK_SECRET')

def verify_signature(payload: bytes, signature: str, timestamp: str) -> bool:
    """Verify HMAC-SHA256 signature."""
    signed_payload = f"{timestamp}.{payload.decode('utf-8')}"
    expected = hmac.new(
        WEBHOOK_SECRET.encode('utf-8'),
        signed_payload.encode('utf-8'),
        hashlib.sha256
    ).hexdigest()

    return hmac.compare_digest(signature, expected)

def verify_webhook(f):
    """Decorator for webhook verification."""
    @wraps(f)
    def decorated(*args, **kwargs):
        signature = request.headers.get('X-Lilith-Signature')
        timestamp = request.headers.get('X-Lilith-Timestamp')
        event_id = request.headers.get('X-Lilith-Event-Id')

        if not all([signature, timestamp, event_id]):
            return jsonify({'error': 'Missing headers'}), 401

        # Verify timestamp
        current_time = int(time.time())
        webhook_time = int(timestamp)

        if abs(current_time - webhook_time) > 300:
            return jsonify({'error': 'Timestamp expired'}), 401

        # Verify signature
        if not verify_signature(request.data, signature, timestamp):
            return jsonify({'error': 'Invalid signature'}), 401

        return f(*args, **kwargs)

    return decorated

@app.route('/webhooks/lilith', methods=['POST'])
@verify_webhook
def handle_webhook():
    event_id = request.headers.get('X-Lilith-Event-Id')

    # Check for duplicate
    if is_duplicate_event(event_id):
        return jsonify({'received': True, 'duplicate': True}), 200

    event = request.get_json()

    # Process asynchronously (using Celery, RQ, etc.)
    process_webhook_event.delay(event, event_id)

    return jsonify({'received': True}), 200

def is_duplicate_event(event_id: str) -> bool:
    """Check Redis for duplicate event."""
    key = f"webhook:{event_id}"
    exists = redis_client.exists(key)
    if not exists:
        redis_client.setex(key, 86400, 'processing')
    return exists

# Celery task for async processing
@celery.task
def process_webhook_event(event: dict, event_id: str):
    event_type = event.get('type')
    data = event.get('data')

    handlers = {
        'content.created': handle_content_created,
        'content.updated': handle_content_updated,
        'payment.succeeded': handle_payment_succeeded,
    }

    handler = handlers.get(event_type)
    if handler:
        handler(data)
    else:
        logger.warning(f"Unhandled event type: {event_type}")

Go Implementation#

go
package main

import (
    "crypto/hmac"
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "io"
    "log"
    "math"
    "net/http"
    "strconv"
    "time"
)

var webhookSecret = []byte(os.Getenv("LILITH_WEBHOOK_SECRET"))

type WebhookEvent struct {
    ID        string          `json:"id"`
    Type      string          `json:"type"`
    Timestamp string          `json:"timestamp"`
    Data      json.RawMessage `json:"data"`
}

func verifySignature(payload []byte, signature, timestamp string) bool {
    signedPayload := timestamp + "." + string(payload)
    h := hmac.New(sha256.New, webhookSecret)
    h.Write([]byte(signedPayload))
    expected := hex.EncodeToString(h.Sum(nil))

    return hmac.Equal([]byte(signature), []byte(expected))
}

func webhookHandler(w http.ResponseWriter, r *http.Request) {
    signature := r.Header.Get("X-Lilith-Signature")
    timestamp := r.Header.Get("X-Lilith-Timestamp")
    eventID := r.Header.Get("X-Lilith-Event-Id")

    // Verify timestamp
    ts, err := strconv.ParseInt(timestamp, 10, 64)
    if err != nil {
        http.Error(w, "Invalid timestamp", http.StatusUnauthorized)
        return
    }

    currentTime := time.Now().Unix()
    if math.Abs(float64(currentTime-ts)) > 300 {
        http.Error(w, "Timestamp expired", http.StatusUnauthorized)
        return
    }

    // Read body
    body, err := io.ReadAll(r.Body)
    if err != nil {
        http.Error(w, "Failed to read body", http.StatusBadRequest)
        return
    }

    // Verify signature
    if !verifySignature(body, signature, timestamp) {
        http.Error(w, "Invalid signature", http.StatusUnauthorized)
        return
    }

    // Check for duplicate
    if isDuplicate(eventID) {
        json.NewEncoder(w).Encode(map[string]interface{}{
            "received":  true,
            "duplicate": true,
        })
        return
    }

    // Acknowledge immediately
    w.WriteHeader(http.StatusOK)
    json.NewEncoder(w).Encode(map[string]bool{"received": true})

    // Process asynchronously
    var event WebhookEvent
    json.Unmarshal(body, &event)
    go processEvent(event, eventID)
}

func processEvent(event WebhookEvent, eventID string) {
    defer func() {
        if r := recover(); r != nil {
            log.Printf("Event processing panic: %v, eventID: %s", r, eventID)
        }
    }()

    switch event.Type {
    case "content.created":
        handleContentCreated(event.Data)
    case "content.updated":
        handleContentUpdated(event.Data)
    case "payment.succeeded":
        handlePaymentSucceeded(event.Data)
    default:
        log.Printf("Unhandled event type: %s", event.Type)
    }
}

func main() {
    http.HandleFunc("/webhooks/lilith", webhookHandler)
    log.Fatal(http.ListenAndServeTLS(":443", "cert.pem", "key.pem", nil))
}

Registering Webhooks#

Create a Webhook#

bash
curl -X POST "https://api.lilith.app/v1/webhooks" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://your-app.com/webhooks/lilith",
    "events": ["content.created", "content.updated"],
    "description": "Production content notifications",
    "metadata": {
      "environment": "production",
      "version": "1.0.0"
    }
  }'

Response:

json
{
  "id": "wh_abc123def456",
  "url": "https://your-app.com/webhooks/lilith",
  "events": ["content.created", "content.updated"],
  "description": "Production content notifications",
  "secret": "whsec_a1b2c3d4e5f6g7h8i9j0...",
  "active": true,
  "created_at": "2025-01-15T10:30:00Z"
}

Important: Store the secret immediately - it's only shown once!

List Webhooks#

bash
curl -X GET "https://api.lilith.app/v1/webhooks" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN"

Update a Webhook#

bash
curl -X PATCH "https://api.lilith.app/v1/webhooks/wh_abc123def456" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "events": ["content.created", "content.updated", "content.deleted"],
    "active": true
  }'

Delete a Webhook#

bash
curl -X DELETE "https://api.lilith.app/v1/webhooks/wh_abc123def456" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN"

Rotate Webhook Secret#

bash
curl -X POST "https://api.lilith.app/v1/webhooks/wh_abc123def456/rotate-secret" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN"

Handling Webhook Events#

Event Payload Structure#

All webhook events follow this structure:

json
{
  "id": "evt_xyz789",
  "type": "content.created",
  "timestamp": "2025-01-15T10:30:00Z",
  "api_version": "2025-01-01",
  "data": {
    "id": "content_abc123",
    "title": "New Meditation Course",
    "status": "published"
  },
  "metadata": {
    "correlation_id": "req_123456"
  }
}

Event Handlers by Type#

javascript
// handlers/contentHandlers.js
async function handleContentCreated(data) {
  const { id, title, type, status } = data;

  // Update local database
  await db.content.create({
    externalId: id,
    title,
    type,
    status,
    syncedAt: new Date(),
  });

  // Notify relevant systems
  await notificationService.send({
    type: 'new_content',
    contentId: id,
    title,
  });

  // Invalidate caches
  await cache.invalidate(`content:list:*`);
}

async function handleContentUpdated(data) {
  const { id, changes } = data;

  // Update local copy
  await db.content.update({
    where: { externalId: id },
    data: {
      ...changes,
      updatedAt: new Date(),
    },
  });

  // Invalidate specific cache
  await cache.invalidate(`content:${id}`);
}

async function handleContentDeleted(data) {
  const { id } = data;

  // Soft delete or remove
  await db.content.delete({
    where: { externalId: id },
  });

  // Clean up related data
  await cleanupRelatedContent(id);
}

// handlers/paymentHandlers.js
async function handlePaymentSucceeded(data) {
  const { userId, amount, subscriptionId, invoiceId } = data;

  // Update subscription status
  await db.subscription.update({
    where: { externalId: subscriptionId },
    data: { status: 'active', lastPaymentAt: new Date() },
  });

  // Grant entitlements
  await entitlementService.grant(userId, subscriptionId);

  // Send confirmation email
  await emailService.send({
    template: 'payment_confirmation',
    to: await getUserEmail(userId),
    data: { amount, invoiceId },
  });
}

async function handlePaymentFailed(data) {
  const { userId, subscriptionId, failureReason } = data;

  // Update status
  await db.subscription.update({
    where: { externalId: subscriptionId },
    data: { status: 'past_due' },
  });

  // Notify user
  await emailService.send({
    template: 'payment_failed',
    to: await getUserEmail(userId),
    data: { reason: failureReason },
  });

  // Start dunning process
  await dunningService.start(userId, subscriptionId);
}

Event Router Pattern#

javascript
class WebhookEventRouter {
  constructor() {
    this.handlers = new Map();
  }

  register(eventType, handler) {
    if (!this.handlers.has(eventType)) {
      this.handlers.set(eventType, []);
    }
    this.handlers.get(eventType).push(handler);
  }

  async route(event) {
    const handlers = this.handlers.get(event.type) || [];

    if (handlers.length === 0) {
      console.warn(`No handlers registered for event: ${event.type}`);
      return;
    }

    const results = await Promise.allSettled(
      handlers.map((handler) => handler(event.data, event))
    );

    const failures = results.filter((r) => r.status === 'rejected');
    if (failures.length > 0) {
      console.error(
        `${failures.length} handler(s) failed for event ${event.id}`
      );
      failures.forEach((f) => console.error(f.reason));
    }
  }
}

// Usage
const router = new WebhookEventRouter();

router.register('content.created', handleContentCreated);
router.register('content.created', syncToSearchIndex);
router.register('content.created', notifySubscribers);

router.register('payment.succeeded', handlePaymentSucceeded);
router.register('payment.succeeded', updateAnalytics);

// In webhook endpoint
app.post('/webhooks/lilith', async (req, res) => {
  // ... verification code ...

  const event = JSON.parse(req.body);
  res.status(200).json({ received: true });

  await router.route(event);
});

Signature Verification#

Why Signature Verification Matters#

Without signature verification, anyone could send fake webhook events to your endpoint. Always verify:

  1. Signature - Proves the request came from Lilith
  2. Timestamp - Prevents replay attacks

Verification Algorithm#

The signature is generated using HMAC-SHA256:

text
signature = HMAC-SHA256(webhook_secret, timestamp + "." + raw_body)

Verification Examples#

Node.js#

javascript
const crypto = require('crypto');

function verifyWebhookSignature(rawBody, signature, timestamp, secret) {
  // Construct signed payload
  const signedPayload = `${timestamp}.${rawBody}`;

  // Compute expected signature
  const expectedSignature = crypto
    .createHmac('sha256', secret)
    .update(signedPayload, 'utf8')
    .digest('hex');

  // Timing-safe comparison
  return crypto.timingSafeEqual(
    Buffer.from(signature),
    Buffer.from(expectedSignature)
  );
}

Python#

python
import hmac
import hashlib

def verify_webhook_signature(raw_body: bytes, signature: str, timestamp: str, secret: str) -> bool:
    signed_payload = f"{timestamp}.{raw_body.decode('utf-8')}"
    expected_signature = hmac.new(
        secret.encode('utf-8'),
        signed_payload.encode('utf-8'),
        hashlib.sha256
    ).hexdigest()

    return hmac.compare_digest(signature, expected_signature)

Go#

go
func verifyWebhookSignature(rawBody []byte, signature, timestamp string, secret []byte) bool {
    signedPayload := timestamp + "." + string(rawBody)
    h := hmac.New(sha256.New, secret)
    h.Write([]byte(signedPayload))
    expected := hex.EncodeToString(h.Sum(nil))

    return hmac.Equal([]byte(signature), []byte(expected))
}

Idempotency and Retry Handling#

Why Idempotency Matters#

Webhooks may be delivered multiple times due to:

  • Network timeouts
  • Your endpoint returning errors
  • Our retry mechanism

Your handlers must be idempotent - processing the same event multiple times should have the same effect as processing it once.

Implementing Idempotency#

javascript
class IdempotentEventProcessor {
  constructor(redis) {
    this.redis = redis;
    this.TTL = 86400; // 24 hours
  }

  async process(eventId, processor) {
    const key = `webhook:event:${eventId}`;

    // Try to acquire lock
    const acquired = await this.redis.set(
      key,
      'processing',
      'EX',
      this.TTL,
      'NX'
    );

    if (!acquired) {
      // Event already being processed or completed
      const status = await this.redis.get(key);
      return { duplicate: true, status };
    }

    try {
      await processor();
      await this.redis.set(key, 'completed', 'EX', this.TTL);
      return { success: true };
    } catch (error) {
      await this.redis.set(key, 'failed', 'EX', this.TTL);
      throw error;
    }
  }
}

// Usage
const processor = new IdempotentEventProcessor(redis);

app.post('/webhooks/lilith', async (req, res) => {
  const event = JSON.parse(req.body);
  const eventId = req.headers['x-lilith-event-id'];

  res.status(200).json({ received: true });

  const result = await processor.process(eventId, async () => {
    await handleEvent(event);
  });

  if (result.duplicate) {
    console.log(`Duplicate event ${eventId}, status: ${result.status}`);
  }
});

Idempotent Database Operations#

javascript
// Bad: Not idempotent - creates duplicates
async function handleContentCreated(data) {
  await db.content.create({ externalId: data.id, title: data.title });
}

// Good: Idempotent - upsert operation
async function handleContentCreated(data) {
  await db.content.upsert({
    where: { externalId: data.id },
    create: { externalId: data.id, title: data.title },
    update: { title: data.title, updatedAt: new Date() },
  });
}

// Bad: Not idempotent - sends multiple emails
async function handlePaymentSucceeded(data) {
  await sendConfirmationEmail(data.userId, data.amount);
}

// Good: Idempotent - check before sending
async function handlePaymentSucceeded(data) {
  const emailSent = await db.emailLog.findFirst({
    where: {
      eventId: data.eventId,
      type: 'payment_confirmation',
    },
  });

  if (!emailSent) {
    await sendConfirmationEmail(data.userId, data.amount);
    await db.emailLog.create({
      eventId: data.eventId,
      type: 'payment_confirmation',
      sentAt: new Date(),
    });
  }
}

Retry Behavior#

Lilith retries failed webhook deliveries with exponential backoff:

Attempt Delay
1 Immediate
2 1 minute
3 5 minutes
4 30 minutes
5 2 hours
6 8 hours
7 24 hours

After 7 failed attempts, the webhook is marked as failed and no further retries occur.

Testing Webhooks#

Using the Test Endpoint#

Send a test event to verify your webhook endpoint:

bash
curl -X POST "https://api.lilith.app/v1/webhooks/wh_abc123def456/test" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "type": "content.created",
    "data": {
      "id": "test_content_123",
      "title": "Test Content",
      "status": "published"
    }
  }'

Local Development with ngrok#

For local testing, use ngrok to expose your local server:

bash
# Start your local server
npm run dev  # Running on localhost:3000

# In another terminal, start ngrok
ngrok http 3000

Use the ngrok URL when registering your webhook:

bash
curl -X POST "https://api.lilith.app/v1/webhooks" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://abc123.ngrok.io/webhooks/lilith",
    "events": ["content.created"]
  }'

Unit Testing Webhook Handlers#

javascript
const { verifySignature, handleContentCreated } = require('./webhooks');

describe('Webhook Handler', () => {
  describe('verifySignature', () => {
    const secret = 'test_secret';
    const timestamp = '1704067200';
    const body = '{"type":"content.created"}';

    it('should verify valid signature', () => {
      const signature = crypto
        .createHmac('sha256', secret)
        .update(`${timestamp}.${body}`)
        .digest('hex');

      expect(verifySignature(body, signature, timestamp, secret)).toBe(true);
    });

    it('should reject invalid signature', () => {
      expect(verifySignature(body, 'invalid', timestamp, secret)).toBe(false);
    });
  });

  describe('handleContentCreated', () => {
    beforeEach(() => {
      jest.clearAllMocks();
    });

    it('should create content in database', async () => {
      const data = { id: 'content_123', title: 'Test', status: 'published' };

      await handleContentCreated(data);

      expect(db.content.upsert).toHaveBeenCalledWith({
        where: { externalId: 'content_123' },
        create: expect.objectContaining({ title: 'Test' }),
        update: expect.objectContaining({ title: 'Test' }),
      });
    });
  });
});

Integration Testing#

javascript
const request = require('supertest');
const crypto = require('crypto');
const app = require('./app');

describe('Webhook Endpoint', () => {
  const webhookSecret = process.env.LILITH_WEBHOOK_SECRET;

  function createSignature(body, timestamp) {
    return crypto
      .createHmac('sha256', webhookSecret)
      .update(`${timestamp}.${JSON.stringify(body)}`)
      .digest('hex');
  }

  it('should accept valid webhook', async () => {
    const timestamp = Math.floor(Date.now() / 1000).toString();
    const body = { type: 'content.created', data: { id: 'test' } };
    const signature = createSignature(body, timestamp);

    const response = await request(app)
      .post('/webhooks/lilith')
      .set('X-Lilith-Signature', signature)
      .set('X-Lilith-Timestamp', timestamp)
      .set('X-Lilith-Event-Id', 'evt_test123')
      .send(body);

    expect(response.status).toBe(200);
    expect(response.body.received).toBe(true);
  });

  it('should reject invalid signature', async () => {
    const response = await request(app)
      .post('/webhooks/lilith')
      .set('X-Lilith-Signature', 'invalid')
      .set('X-Lilith-Timestamp', Date.now().toString())
      .set('X-Lilith-Event-Id', 'evt_test123')
      .send({ type: 'content.created' });

    expect(response.status).toBe(401);
  });

  it('should reject expired timestamp', async () => {
    const oldTimestamp = (Math.floor(Date.now() / 1000) - 600).toString();
    const body = { type: 'content.created' };
    const signature = createSignature(body, oldTimestamp);

    const response = await request(app)
      .post('/webhooks/lilith')
      .set('X-Lilith-Signature', signature)
      .set('X-Lilith-Timestamp', oldTimestamp)
      .set('X-Lilith-Event-Id', 'evt_test123')
      .send(body);

    expect(response.status).toBe(401);
  });
});

Production Best Practices#

1. Response Quickly#

Always return 200 immediately, then process asynchronously:

javascript
// Good: Acknowledge then process
app.post('/webhooks/lilith', (req, res) => {
  res.status(200).json({ received: true });
  processAsync(req.body); // Returns immediately
});

// Bad: Process then acknowledge
app.post('/webhooks/lilith', async (req, res) => {
  await longRunningProcess(req.body); // May timeout
  res.status(200).json({ received: true });
});

2. Use Message Queues#

For reliable processing, use a message queue:

javascript
const { Queue } = require('bullmq');
const webhookQueue = new Queue('webhooks');

app.post('/webhooks/lilith', async (req, res) => {
  // ... verification ...

  await webhookQueue.add('process-event', {
    event: JSON.parse(req.body),
    eventId: req.headers['x-lilith-event-id'],
    receivedAt: Date.now(),
  });

  res.status(200).json({ received: true });
});

// Worker
const { Worker } = require('bullmq');

new Worker('webhooks', async (job) => {
  const { event, eventId } = job.data;
  await processEvent(event, eventId);
});

3. Monitor Webhook Health#

Track webhook delivery metrics:

javascript
const metrics = require('./metrics');

app.post('/webhooks/lilith', (req, res) => {
  const startTime = Date.now();
  const eventType = JSON.parse(req.body).type;

  metrics.counter('webhook.received', { type: eventType });

  // ... processing ...

  metrics.histogram('webhook.processing_time', Date.now() - startTime);
  res.status(200).json({ received: true });
});

4. Handle Failures Gracefully#

javascript
async function processWithRetry(event, maxRetries = 3) {
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      await processEvent(event);
      return;
    } catch (error) {
      if (attempt === maxRetries) {
        // Log to dead letter queue for manual review
        await deadLetterQueue.add(event, error);
        throw error;
      }

      const delay = Math.pow(2, attempt) * 1000;
      await sleep(delay);
    }
  }
}

5. Secure Your Endpoint#

  • Use HTTPS only
  • Verify signatures on every request
  • Check timestamp freshness
  • Rate limit incoming requests
  • Log all webhook activity
javascript
const rateLimit = require('express-rate-limit');

const webhookLimiter = rateLimit({
  windowMs: 60 * 1000, // 1 minute
  max: 100, // 100 requests per minute
  message: 'Too many webhook requests',
});

app.post(
  '/webhooks/lilith',
  webhookLimiter,
  rawBodyMiddleware,
  async (req, res) => {
    // ... handler ...
  }
);

Troubleshooting#

Common Issues#

Signature Verification Fails#

  1. Wrong secret - Ensure you're using the correct webhook secret
  2. Body parsing - Use raw body for signature verification
  3. Encoding issues - Ensure UTF-8 encoding throughout
javascript
// Debug signature verification
function debugVerification(rawBody, signature, timestamp, secret) {
  console.log('Raw body:', rawBody);
  console.log('Raw body type:', typeof rawBody);
  console.log('Signature:', signature);
  console.log('Timestamp:', timestamp);

  const signedPayload = `${timestamp}.${rawBody}`;
  console.log('Signed payload:', signedPayload);

  const expected = crypto
    .createHmac('sha256', secret)
    .update(signedPayload)
    .digest('hex');
  console.log('Expected signature:', expected);
  console.log('Match:', signature === expected);
}

Missing Events#

  1. Check webhook status - Ensure webhook is active
  2. Verify event subscription - Check if subscribed to event type
  3. Review delivery history - Check for failed deliveries
bash
# Get webhook details
curl -X GET "https://api.lilith.app/v1/webhooks/wh_abc123" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN"

# Get delivery history
curl -X GET "https://api.lilith.app/v1/webhooks/wh_abc123/deliveries" \
  -H "Authorization: Bearer YOUR_JWT_TOKEN"

Timeout Errors#

Your endpoint must respond within 30 seconds. If processing takes longer:

  1. Return 200 immediately
  2. Process asynchronously
  3. Use a message queue for reliability

Duplicate Events#

Implement idempotency using event IDs:

javascript
const processedEvents = new Set(); // Use Redis in production

app.post('/webhooks/lilith', (req, res) => {
  const eventId = req.headers['x-lilith-event-id'];

  if (processedEvents.has(eventId)) {
    return res.status(200).json({ received: true, duplicate: true });
  }

  processedEvents.add(eventId);
  // ... process event ...
});

Getting Help#

If you're experiencing webhook issues:

  1. Check the Developer Dashboard for delivery logs
  2. Review the Error Codes Reference
  3. Contact API Support with:
    • Webhook ID
    • Event IDs that failed
    • Error messages
    • Your endpoint logs