${message}`
});
}
async function checkAndAlert() {
const stats = await getFailureRate(SUBSCRIPTION_ID);
if (stats.failureRate > 20) {
const failedAttempts = stats.attempts.filter(a => a.failed_at !== null);
await sendAlert('High Webhook Failure Rate', `
Failure Rate: ${stats.failureRate.toFixed(2)}%
Total Attempts: ${stats.total}
Failed Attempts: ${stats.failed}
Recent failures:
${JSON.stringify(failedAttempts, null, 2)}
`);
}
}
```
## Step 6: Recovery strategies
### Manual retry of failed events
```bash
# Get events for your application
curl "$HOOK0_API/events/?application_id=$APP_ID" \
-H "Authorization: Bearer $HOOK0_TOKEN"
# Replay a specific event
curl -X POST "$HOOK0_API/events/{EVENT_ID}/replay" \
-H "Authorization: Bearer $HOOK0_TOKEN" \
-H "Content-Type: application/json" \
-d '{
"application_id": "'"$APP_ID"'"
}'
```
### Bulk retry script
```javascript
// retry-failed.js
const HOOK0_TOKEN = '{YOUR_TOKEN}';
const APP_ID = '{APP_ID}';
async function retryFailedEvents(maxAge = 24) {
// Get failed request attempts
const response = await fetch(
`http://localhost:8081/api/v1/request_attempts/?application_id=${APP_ID}`,
{
headers: { 'Authorization': `Bearer ${HOOK0_TOKEN}` }
}
);
const attempts = await response.json();
// Filter failed attempts (failed_at is set when delivery failed)
const failedAttempts = attempts.filter(a => a.failed_at !== null);
// Get unique event IDs
const uniqueEvents = [...new Set(failedAttempts.map(a => a.event_id))];
console.log(`Found ${uniqueEvents.length} failed events to retry`);
for (const eventId of uniqueEvents) {
try {
const retryResponse = await fetch(
`http://localhost:8081/api/v1/events/${eventId}/replay`,
{
method: 'POST',
headers: {
'Authorization': `Bearer ${HOOK0_TOKEN}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({ application_id: APP_ID })
}
);
if (retryResponse.ok) {
console.log(`✅ Replayed event: ${eventId}`);
} else {
const error = await retryResponse.text();
console.log(`❌ Failed to replay event: ${eventId} - ${error}`);
}
// Rate limit retries
await new Promise(resolve => setTimeout(resolve, 100));
} catch (error) {
console.error(`Error replaying event ${eventId}:`, error.message);
}
}
}
retryFailedEvents();
```
## Step 7: Prevention strategies
### Circuit breaker pattern
```javascript
// circuit-breaker.js
class CircuitBreaker {
constructor(threshold = 5, timeout = 60000) {
this.threshold = threshold;
this.timeout = timeout;
this.failureCount = 0;
this.lastFailure = null;
this.state = 'CLOSED'; // CLOSED, OPEN, HALF_OPEN
}
async call(fn) {
if (this.state === 'OPEN') {
if (Date.now() - this.lastFailure < this.timeout) {
throw new Error('Circuit breaker is OPEN');
}
this.state = 'HALF_OPEN';
}
try {
const result = await fn();
this.onSuccess();
return result;
} catch (error) {
this.onFailure();
throw error;
}
}
onSuccess() {
this.failureCount = 0;
this.state = 'CLOSED';
}
onFailure() {
this.failureCount++;
this.lastFailure = Date.now();
if (this.failureCount >= this.threshold) {
this.state = 'OPEN';
}
}
}
// Use in webhook handler
const circuitBreaker = new CircuitBreaker(3, 30000);
app.post('/webhook', async (req, res) => {
try {
await circuitBreaker.call(async () => {
await processWebhook(req.body);
});
res.json({ status: 'processed' });
} catch (error) {
console.error('Circuit breaker prevented processing:', error.message);
res.status(503).json({ error: 'Service temporarily unavailable' });
}
});
```
### Graceful degradation
```javascript
// graceful-degradation.js
app.post('/webhook', async (req, res) => {
try {
// Try primary processing
await processPrimary(req.body);
res.json({ status: 'processed' });
} catch (primaryError) {
console.warn('Primary processing failed, trying fallback:', primaryError.message);
try {
// Try fallback processing
await processFallback(req.body);
res.json({ status: 'processed_fallback' });
} catch (fallbackError) {
console.error('Both primary and fallback failed:', fallbackError.message);
// Store for later processing
await storeForRetry(req.body);
res.status(202).json({ status: 'queued_for_retry' });
}
}
});
```
## Best practices
### Endpoint design
- ✅ Return appropriate HTTP status codes
- ✅ Respond within 30 seconds
- ✅ Implement idempotency
- ✅ Use structured error responses
- ✅ Add comprehensive logging
### Error handling
- ✅ Distinguish between temporary and permanent failures
- ✅ Rely on Hook0's automatic retries with predefined delays
- ✅ Use circuit breakers for external dependencies
- ✅ Monitor and alert on high failure rates
- ✅ Provide detailed error messages
### Testing
- ✅ Test webhook endpoints thoroughly
- ✅ Simulate failure scenarios
- ✅ Verify signature validation
- ✅ Test with various payload sizes
- ✅ Monitor performance under load
## Troubleshooting checklist
When webhook deliveries fail, work through this systematic checklist:
### Infrastructure
- [ ] Endpoint is accessible from internet
- [ ] SSL certificate is valid and not expired
- [ ] DNS resolution works correctly
- [ ] Firewall allows incoming connections
- [ ] Load balancer is configured correctly
### Application
- [ ] Webhook handler is properly implemented
- [ ] Signature verification is working
- [ ] Response times are under 30 seconds
- [ ] Error handling returns appropriate status codes
- [ ] Logging captures enough detail for debugging
### Hook0 Configuration
- [ ] Subscription is enabled and active (see [Events Not Being Delivered](./troubleshooting.md#events-not-being-delivered))
- [ ] Event types match what you're sending
- [ ] Target URL is correct
- [ ] Custom headers are properly configured
- [ ] Retry configuration is appropriate
## Related resources
- [Troubleshooting Guide](./troubleshooting.md) - API errors, authentication, and configuration issues
- [Implementing Webhook Authentication](../tutorials/webhook-authentication.md) - Signature verification guide
- [Monitor Webhook Performance](./monitor-webhook-performance.md) - Performance monitoring
- [Error Codes Reference](../reference/error-codes.md) - Error code list
# Securing webhook endpoints
A webhook endpoint is secure when it rejects every request it cannot prove came from the sender. With Hook0 that means five checks, in this order: verify the HMAC-SHA256 signature in the `X-Hook0-Signature` header against the raw body, reject timestamps outside a short window (300 seconds is a sensible default), validate the payload against a schema, skip any `event_id` you have already processed, and rate-limit the route. An IP allowlist is not one of them: Hook0 does not guarantee fixed delivery IPs, so an allowlist ends up dropping real webhooks.
This guide implements each check, using [subscription signing secrets](/concepts/subscriptions#subscription-secrets) and [subscription](/concepts/subscriptions) configuration, then adds logging and attack detection.
## IP allowlist or HMAC signature?
| | IP allowlist | HMAC signature |
|-|--------------|----------------|
| Proves the request came from Hook0 | No, only where it came from | Yes, only the holder of the subscription secret can sign |
| Detects a modified payload | No | Yes, the signature covers the body |
| Blocks a replayed request | No | Yes, combined with the timestamp check |
| Survives sender infrastructure changes | No, breaks when delivery IPs change | Yes, independent of the network |
| Secret rotation | Not applicable | Accept the old and new secret during the overlap |
Keep IP filtering for blocking abusive sources (Step 4), not as proof of origin.
:::tip Test your verifier against real signatures
Send a request to [Hook0 Play](https://play.hook0.com) and copy its `X-Hook0-Signature` header, or [create a free Hook0 Cloud account](https://app.hook0.com/register) (100 events a day, no credit card) to sign deliveries with your own subscription secret.
:::
## Quick Start (5 minutes)
1. **Verify HMAC signature** - Check `X-Hook0-Signature` header
2. **Validate payload** - Use Joi/Zod to validate JSON structure
3. **Implement idempotency** - Track processed `event_id` to avoid duplicates
4. **Log requests** - Record all webhook attempts for debugging
For detailed implementation, see sections below.
## Security threat model
### Common webhook security threats
:::danger Spoofing attacks
Attackers sending fake webhook requests.
**Mitigation**: Signature verification. IP allowlisting does not prove origin, see [IP allowlist or HMAC signature?](#ip-allowlist-or-hmac-signature)
:::
:::warning Replay attacks
Reusing legitimate webhook requests.
**Mitigation**: Timestamp validation, nonce tracking
:::
:::info Man-in-the-Middle
Intercepting and modifying webhook data.
**Mitigation**: HTTPS, certificate pinning
:::
:::danger Denial of service
Overwhelming webhook endpoints with requests.
**Mitigation**: Rate limiting, request validation
:::
:::warning Data injection
Malicious payloads causing application vulnerabilities.
**Mitigation**: Input validation, sanitization
:::
## Step 1: Signature verification
Hook0 uses HMAC-SHA256 signatures with the v1 format. For complete signature verification implementation in JavaScript, Python, and Go, see [Implementing Webhook Authentication](../tutorials/webhook-authentication.md).
Key security considerations for signature verification:
- Use timing-safe comparison to prevent timing attacks
- Support multiple secrets for rotation
- Validate timestamp to prevent replay attacks
- Use raw request body (not parsed JSON)
While implementing, send a test webhook to a [Hook0 Play](https://play.hook0.com) URL and copy the `X-Hook0-Signature` header value to verify your signature computation.
## Step 2: Input validation
### Schema-based validation
```javascript
// webhook-validator.js
const Joi = require('joi');
// Define webhook payload schemas
const eventSchemas = {
'user.account.created': Joi.object({
event_id: Joi.string().uuid().required(),
event_type: Joi.string().valid('user.account.created').required(),
timestamp: Joi.string().isoDate().required(),
payload: Joi.object({
user_id: Joi.string().required(),
email: Joi.string().email().required(),
name: Joi.string().min(1).max(255).required(),
plan: Joi.string().optional()
}).required(),
labels: Joi.object().optional()
}),
'order.completed': Joi.object({
event_id: Joi.string().uuid().required(),
event_type: Joi.string().valid('order.completed').required(),
timestamp: Joi.string().isoDate().required(),
payload: Joi.object({
order_id: Joi.string().required(),
customer_id: Joi.string().required(),
total_amount: Joi.number().positive().required(),
currency: Joi.string().length(3).uppercase().required()
}).required(),
labels: Joi.object().optional()
})
};
function validateWebhookPayload(payload) {
const schema = eventSchemas[payload.event_type];
if (!schema) {
return {
valid: false,
error: `Unknown event type: ${payload.event_type}`
};
}
const { error, value } = schema.validate(payload, {
abortEarly: false,
stripUnknown: true
});
if (error) {
return {
valid: false,
error: 'Validation failed',
details: error.details.map(d => ({
field: d.path.join('.'),
message: d.message
}))
};
}
return { valid: true, data: value };
}
module.exports = { validateWebhookPayload };
```
### Sanitization and security filters
```javascript
// security-filters.js
const DOMPurify = require('isomorphic-dompurify');
const validator = require('validator');
class SecurityFilters {
static sanitizeString(input, options = {}) {
if (typeof input !== 'string') return input;
let sanitized = input;
// HTML sanitization
if (options.allowHtml) {
sanitized = DOMPurify.sanitize(sanitized);
} else {
sanitized = validator.escape(sanitized);
}
// Length limits
if (options.maxLength) {
sanitized = sanitized.substring(0, options.maxLength);
}
// Remove control characters
sanitized = sanitized.replace(/[\x00-\x1F\x7F]/g, '');
return sanitized;
}
static validateEmail(email) {
return validator.isEmail(email) && email.length <= 254;
}
static validateURL(url) {
return validator.isURL(url, {
protocols: ['https'],
require_protocol: true,
require_valid_protocol: true
});
}
static sanitizePayload(payload, rules = {}) {
const sanitized = {};
for (const [key, value] of Object.entries(payload)) {
const rule = rules[key] || {};
if (typeof value === 'string') {
sanitized[key] = this.sanitizeString(value, rule);
} else if (typeof value === 'object' && value !== null) {
sanitized[key] = this.sanitizePayload(value, rule.nested || {});
} else {
sanitized[key] = value;
}
}
return sanitized;
}
}
module.exports = SecurityFilters;
```
## Step 3: Request deduplication
### Idempotency with in-memory storage
```javascript
// idempotency-manager.js
class IdempotencyManager {
constructor(options = {}) {
this.store = new Map();
this.defaultTTL = options.ttl || 24 * 60 * 60 * 1000; // 24 hours in ms
this.maxSize = options.maxSize || 10000;
// Cleanup old entries periodically
this.cleanupInterval = setInterval(() => this.cleanup(), 60 * 60 * 1000); // Every hour
}
generateKey(eventId, subscriptionId) {
return `${subscriptionId}:${eventId}`;
}
async checkAndSetProcessed(eventId, subscriptionId, ttl = this.defaultTTL) {
const key = this.generateKey(eventId, subscriptionId);
const existing = this.store.get(key);
if (existing) {
// Key already exists, this is a duplicate
return {
isFirst: false,
key,
previousProcessing: existing.data
};
}
// Key doesn't exist, set it
this.store.set(key, {
data: {
processedAt: new Date().toISOString(),
eventId,
subscriptionId
},
expiresAt: Date.now() + ttl
});
// Enforce max size
if (this.store.size > this.maxSize) {
this.cleanup();
}
return { isFirst: true, key };
}
async markCompleted(eventId, subscriptionId, result) {
const key = this.generateKey(eventId, subscriptionId);
const entry = this.store.get(key);
if (entry) {
entry.data = {
...entry.data,
completedAt: new Date().toISOString(),
result: result,
status: 'completed'
};
}
}
async markFailed(eventId, subscriptionId, error) {
const key = this.generateKey(eventId, subscriptionId);
const entry = this.store.get(key);
if (entry) {
entry.data = {
...entry.data,
failedAt: new Date().toISOString(),
error: {
message: error.message,
stack: error.stack
},
status: 'failed'
};
}
}
cleanup() {
const now = Date.now();
let removed = 0;
for (const [key, entry] of this.store.entries()) {
if (entry.expiresAt < now) {
this.store.delete(key);
removed++;
}
}
console.log(`Cleaned up ${removed} expired entries. Current size: ${this.store.size}`);
}
destroy() {
clearInterval(this.cleanupInterval);
this.store.clear();
}
}
// Usage in webhook handler
const idempotency = new IdempotencyManager({
ttl: 24 * 60 * 60 * 1000, // 24 hours
maxSize: 10000
});
app.post('/webhook', async (req, res) => {
const { event_id } = req.webhook;
const subscriptionId = req.headers['hook0-subscription-id'];
try {
const idempotencyCheck = await idempotency.checkAndSetProcessed(
event_id,
subscriptionId
);
if (!idempotencyCheck.isFirst) {
console.log(`Duplicate webhook ignored: ${event_id}`);
return res.json({
status: 'duplicate',
previousProcessing: idempotencyCheck.previousProcessing
});
}
// Process the webhook
const result = await processWebhook(req.webhook);
await idempotency.markCompleted(event_id, subscriptionId, result);
res.json({ status: 'processed', result });
} catch (error) {
await idempotency.markFailed(event_id, subscriptionId, error);
console.error('Webhook processing failed:', error);
res.status(500).json({ error: 'Processing failed' });
}
});
module.exports = IdempotencyManager;
```
:::tip Production note
For production with multiple instances, use PostgreSQL for idempotency storage instead of in-memory. Query the database to check if an event has been processed before.
:::
## Step 4: IP allowlisting and geolocation
:::caution Do not allowlist Hook0 delivery IPs
Hook0 does not guarantee a fixed set of delivery IPs. Use the filter below to block abusive sources or to restrict traffic you control, and keep `requireAllowlist` off for Hook0 webhooks: the HMAC signature from Step 1 is what proves origin.
:::
### IP filtering
```javascript
// ip-security.js
const geoip = require('geoip-lite');
const ipRangeCheck = require('ip-range-check');
class IPSecurity {
constructor(config = {}) {
this.allowedIPs = config.allowedIPs || [];
this.allowedRanges = config.allowedRanges || [];
this.allowedCountries = config.allowedCountries || [];
this.blockedCountries = config.blockedCountries || [];
this.requireAllowlist = config.requireAllowlist || false;
}
isIPAllowed(clientIP) {
// Remove IPv6 wrapper if present
const ip = clientIP.replace(/^::ffff:/, '');
// Check explicit IP allowlist
if (this.allowedIPs.includes(ip)) {
return { allowed: true, reason: 'explicit_allow' };
}
// Check IP ranges
for (const range of this.allowedRanges) {
if (ipRangeCheck(ip, range)) {
return { allowed: true, reason: 'range_allow' };
}
}
// Check geolocation
const geo = geoip.lookup(ip);
if (geo) {
// Check blocked countries first
if (this.blockedCountries.includes(geo.country)) {
return { allowed: false, reason: 'country_blocked', country: geo.country };
}
// Check allowed countries
if (this.allowedCountries.length > 0) {
if (this.allowedCountries.includes(geo.country)) {
return { allowed: true, reason: 'country_allow', country: geo.country };
} else {
return { allowed: false, reason: 'country_not_allowed', country: geo.country };
}
}
}
// If allowlist is required and no explicit allow, block
if (this.requireAllowlist) {
return { allowed: false, reason: 'not_in_allowlist' };
}
// Default allow if no specific rules matched
return { allowed: true, reason: 'default_allow' };
}
middleware() {
return (req, res, next) => {
const clientIP = req.ip ||
req.connection.remoteAddress ||
req.headers['x-forwarded-for']?.split(',')[0]?.trim();
const ipCheck = this.isIPAllowed(clientIP);
// Log all IP checks for monitoring
console.log('IP Security Check:', {
ip: clientIP,
allowed: ipCheck.allowed,
reason: ipCheck.reason,
country: ipCheck.country,
timestamp: new Date().toISOString()
});
if (!ipCheck.allowed) {
return res.status(403).json({
error: 'Access denied',
reason: ipCheck.reason,
country: ipCheck.country
});
}
req.clientIP = clientIP;
req.ipSecurity = ipCheck;
next();
};
}
}
module.exports = IPSecurity;
```
## Step 5: Request logging and monitoring
### Request logging
```javascript
// webhook-logger.js
const winston = require('winston');
const crypto = require('crypto');
class WebhookLogger {
constructor(options = {}) {
this.logger = winston.createLogger({
level: options.level || 'info',
format: winston.format.combine(
winston.format.timestamp(),
winston.format.errors({ stack: true }),
winston.format.json()
),
transports: [
new winston.transports.Console(),
new winston.transports.File({
filename: options.logFile || 'webhooks.log'
})
]
});
}
hashSensitiveData(data) {
return crypto.createHash('sha256').update(JSON.stringify(data)).digest('hex').substring(0, 16);
}
logWebhookRequest(req, res, responseTime) {
const requestId = crypto.randomUUID();
const payloadHash = this.hashSensitiveData(req.webhook);
const logData = {
requestId,
timestamp: new Date().toISOString(),
method: req.method,
path: req.path,
statusCode: res.statusCode,
responseTime,
clientIP: req.clientIP,
userAgent: req.headers['user-agent'],
contentLength: req.headers['content-length'],
payloadHash,
eventType: req.webhook?.event_type,
eventId: req.webhook?.event_id,
signatureValid: req.webhookAuth?.signature?.valid,
timestampValid: req.webhookAuth?.timestamp?.valid,
ipSecurityReason: req.ipSecurity?.reason,
ipSecurityCountry: req.ipSecurity?.country
};
if (res.statusCode >= 400) {
this.logger.error('Webhook request failed', logData);
} else {
this.logger.info('Webhook request processed', logData);
}
return requestId;
}
logSecurityEvent(type, details) {
this.logger.warn('Security event detected', {
type,
details,
timestamp: new Date().toISOString()
});
}
middleware() {
return (req, res, next) => {
const startTime = Date.now();
// Override res.end to capture response time
const originalEnd = res.end;
res.end = (...args) => {
const responseTime = Date.now() - startTime;
this.logWebhookRequest(req, res, responseTime);
originalEnd.apply(res, args);
};
next();
};
}
}
module.exports = WebhookLogger;
```
## Step 6: Attack detection
### Anomaly detection
```javascript
// anomaly-detector.js
class AnomalyDetector {
constructor(options = {}) {
this.suspiciousPatterns = options.suspiciousPatterns || [];
this.rateLimits = options.rateLimits || {
requestsPerMinute: 60,
failuresPerMinute: 10
};
this.monitoring = {
requests: new Map(),
failures: new Map(),
patterns: new Map()
};
}
detectSuspiciousPayload(payload) {
const payloadStr = JSON.stringify(payload).toLowerCase();
const suspiciousPatterns = [
/script\s*>/i,
/