Webhooks

POST   /v1/webhooks                              # 创建 Webhook 订阅
GET    /v1/webhooks                              # 列出所有 Webhooks
GET    /v1/webhooks/:webhookId                   # 获取 Webhook 详情
PUT    /v1/webhooks/:webhookId                   # 更新 Webhook
DELETE /v1/webhooks/:webhookId                   # 删除 Webhook
GET    /v1/webhooks/:webhookId/deliveries        # 获取投递历史
POST   /v1/webhooks/:webhookId/test              # 发送测试 Webhook
PUT    /v1/webhooks/:webhookId/activate          # 激活 Webhook
PUT    /v1/webhooks/:webhookId/deactivate        # 停用 Webhook
POST   /v1/webhooks/:webhookId/deliveries/:deliveryId/replay  # 重放失败的投递
GET    /v1/webhook-events                        # 列出支持的事件

{
  "success": true,
  "data": {
    "webhook_id": "webhook-uuid-here",
    "secret": "a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6q7r8s9t0u1v2w3x4y5z6",
    "message": "Webhook created successfully. Save the secret - it won't be shown again."
  }
}

Content-Type: application/json
X-AnyCrawl-Signature: sha256=abc123...
X-Webhook-Event: scrape.completed
X-Webhook-Delivery-Id: delivery-uuid-1
X-Webhook-Timestamp: 2026-01-27T10:00:00.000Z

{
  "job_id": "job-uuid-1",
  "status": "completed",
  "url": "https://example.com",
  "total": 10,
  "completed": 10,
  "failed": 0,
  "credits_used": 5,
  "created_at": "2026-01-27T09:00:00.000Z",
  "completed_at": "2026-01-27T10:00:00.000Z"
}

{
  "job_id": "job-uuid-1",
  "status": "failed",
  "url": "https://example.com",
  "error_message": "Connection timeout",
  "credits_used": 3,
  "created_at": "2026-01-27T09:00:00.000Z",
  "completed_at": "2026-01-27T10:00:00.000Z"
}

{
  "task_id": "task-uuid-1",
  "task_name": "Daily News Scrape",
  "execution_id": "exec-uuid-1",
  "execution_number": 45,
  "status": "completed",
  "job_id": "job-uuid-1",
  "credits_used": 5,
  "scheduled_for": "2026-01-27T09:00:00.000Z",
  "completed_at": "2026-01-27T09:02:15.000Z"
}

signature = HMAC-SHA256(payload, webhook_secret)
header_value = "sha256=" + hex(signature)

const crypto = require('crypto');
const express = require('express');

function verifyWebhookSignature(payload, signature, secret) {
  const hmac = crypto.createHmac('sha256', secret);
  hmac.update(JSON.stringify(payload));
  const expectedSignature = `sha256=${hmac.digest('hex')}`;

  return crypto.timingSafeEqual(
    Buffer.from(signature),
    Buffer.from(expectedSignature)
  );
}

const app = express();
app.use(express.json());

app.post('/webhooks/anycrawl', (req, res) => {
  const signature = req.headers['x-anycrawl-signature'];
  const secret = process.env.WEBHOOK_SECRET;

  // Verify signature
  if (!verifyWebhookSignature(req.body, signature, secret)) {
    return res.status(401).json({ error: 'Invalid signature' });
  }

  // Extract event info
  const eventType = req.headers['x-webhook-event'];
  const deliveryId = req.headers['x-webhook-delivery-id'];

  console.log(`Received event: ${eventType}`);
  console.log(`Delivery ID: ${deliveryId}`);
  console.log('Payload:', req.body);

  // Respond quickly (< 5 seconds recommended)
  res.status(200).json({ received: true });

  // Process asynchronously
  processWebhookAsync(eventType, req.body).catch(console.error);
});

app.listen(3000);

import hmac
import hashlib
import json
from flask import Flask, request, jsonify

app = Flask(__name__)
WEBHOOK_SECRET = 'your-webhook-secret-here'

def verify_webhook_signature(payload, signature, secret):
    expected_signature = 'sha256=' + hmac.new(
        secret.encode('utf-8'),
        json.dumps(payload).encode('utf-8'),
        hashlib.sha256
    ).hexdigest()

    return hmac.compare_digest(signature, expected_signature)

@app.route('/webhooks/anycrawl', methods=['POST'])
def webhook_handler():
    signature = request.headers.get('X-AnyCrawl-Signature')
    payload = request.get_json()

    # Verify signature
    if not verify_webhook_signature(payload, signature, WEBHOOK_SECRET):
        return jsonify({'error': 'Invalid signature'}), 401

    # Extract event info
    event_type = request.headers.get('X-Webhook-Event')
    delivery_id = request.headers.get('X-Webhook-Delivery-Id')

    print(f'Received event: {event_type}')
    print(f'Delivery ID: {delivery_id}')
    print(f'Payload: {payload}')

    # Respond quickly
    return jsonify({'received': True}), 200

if __name__ == '__main__':
    app.run(port=3000)

package main

import (
    "crypto/hmac"
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "os"
)

func verifyWebhookSignature(payload []byte, signature, secret string) bool {
    mac := hmac.New(sha256.New, []byte(secret))
    mac.Write(payload)
    expectedSignature := "sha256=" + hex.EncodeToString(mac.Sum(nil))
    return hmac.Equal([]byte(signature), []byte(expectedSignature))
}

func webhookHandler(w http.ResponseWriter, r *http.Request) {
    signature := r.Header.Get("X-AnyCrawl-Signature")
    eventType := r.Header.Get("X-Webhook-Event")
    secret := os.Getenv("WEBHOOK_SECRET")

    body, err := io.ReadAll(r.Body)
    if err != nil {
        http.Error(w, "Error reading body", http.StatusBadRequest)
        return
    }

    if !verifyWebhookSignature(body, signature, secret) {
        http.Error(w, "Invalid signature", http.StatusUnauthorized)
        return
    }

    var payload map[string]interface{}
    if err := json.Unmarshal(body, &payload); err != nil {
        http.Error(w, "Invalid JSON", http.StatusBadRequest)
        return
    }

    fmt.Printf("Received event: %s\n", eventType)
    fmt.Printf("Payload: %+v\n", payload)

    w.Header().Set("Content-Type", "application/json")
    json.NewEncoder(w).Encode(map[string]bool{"received": true})
}

func main() {
    http.HandleFunc("/webhooks/anycrawl", webhookHandler)
    http.ListenAndServe(":3000", nil)
}

{
  "success": true,
  "data": [
    {
      "uuid": "webhook-uuid-1",
      "name": "Production Notifications",
      "webhook_url": "https://your-domain.com/webhooks/anycrawl",
      "webhook_secret": "***hidden***",
      "event_types": ["scrape.completed", "scrape.failed"],
      "scope": "all",
      "is_active": true,
      "consecutive_failures": 0,
      "total_deliveries": 145,
      "successful_deliveries": 142,
      "failed_deliveries": 3,
      "last_success_at": "2026-01-27T10:00:00.000Z",
      "last_failure_at": "2026-01-26T15:30:00.000Z",
      "created_at": "2026-01-01T00:00:00.000Z"
    }
  ]
}

{
  "message": "This is a test webhook from AnyCrawl",
  "timestamp": "2026-01-27T10:00:00.000Z",
  "webhook_id": "webhook-uuid-1"
}

{
  "success": true,
  "message": "Delivery replayed successfully",
  "data": {
    "delivery_id": "delivery-uuid-1",
    "status": "pending"
  }
}

{
  "success": true,
  "data": [
    {
      "uuid": "delivery-uuid-1",
      "webhookSubscriptionUuid": "webhook-uuid-1",
      "eventType": "scrape.completed",
      "status": "delivered",
      "attempt_number": 1,
      "request_url": "https://your-domain.com/webhooks/anycrawl",
      "request_method": "POST",
      "response_status": 200,
      "response_duration_ms": 125,
      "created_at": "2026-01-27T10:00:00.000Z",
      "delivered_at": "2026-01-27T10:00:00.125Z"
    },
    {
      "uuid": "delivery-uuid-2",
      "status": "failed",
      "attempt_number": 3,
      "error_message": "Connection timeout",
      "error_code": "ETIMEDOUT",
      "created_at": "2026-01-27T09:00:00.000Z"
    }
  ],
  "meta": {
    "limit": 20,
    "offset": 0,
    "filters": {
      "status": null,
      "from": null,
      "to": null
    }
  }
}

curl -X PUT "https://api.anycrawl.dev/v1/webhooks/:webhookId/activate" \
  -H "Authorization: Bearer <your-api-key>"

{
  "scope": "all",
  "event_types": ["scrape.completed", "crawl.completed"]
}

{
  "scope": "specific",
  "specific_task_ids": ["task-uuid-1", "task-uuid-2"],
  "event_types": ["task.executed", "task.failed"]
}

ALLOW_LOCAL_WEBHOOKS=true

app.post('/webhook', async (req, res) => {
  // Verify signature
  if (!verifySignature(req.body, req.headers['x-anycrawl-signature'])) {
    return res.status(401).send('Invalid signature');
  }

  // Quick acknowledgment
  res.status(200).json({ received: true });

  // Process asynchronously
  queue.add('process-webhook', req.body);
});

const processedDeliveries = new Set();

app.post('/webhook', (req, res) => {
  const deliveryId = req.headers['x-webhook-delivery-id'];

  if (processedDeliveries.has(deliveryId)) {
    return res.status(200).json({ received: true, duplicate: true });
  }

  processedDeliveries.add(deliveryId);

  // Process event...

  res.status(200).json({ received: true });
});

app.post('/webhook', (req, res) => {
  const deliveryId = req.headers['x-webhook-delivery-id'];
  const eventType = req.headers['x-webhook-event'];

  logger.info('Webhook received', {
    deliveryId,
    eventType,
    timestamp: req.headers['x-webhook-timestamp']
  });

  try {
    processWebhook(req.body, eventType);
    logger.info('Webhook processed', { deliveryId });
    res.status(200).json({ received: true });
  } catch (error) {
    logger.error('Webhook failed', {
      deliveryId,
      error: error.message
    });
    res.status(500).json({ error: 'Processing failed' });
  }
});

app.post('/webhooks/anycrawl', async (req, res) => {
  const { job_id, status, url } = req.body;

  await fetch(process.env.SLACK_WEBHOOK_URL, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      text: `Job ${status}: ${url}\nJob ID: ${job_id}`
    })
  });

  res.status(200).json({ received: true });
});

app.post('/webhooks/anycrawl', async (req, res) => {
  const eventType = req.headers['x-webhook-event'];

  if (eventType.endsWith('.failed')) {
    await sendEmail({
      to: 'admin@example.com',
      subject: 'AnyCrawl Job Failed',
      body: JSON.stringify(req.body, null, 2)
    });
  }

  res.status(200).json({ received: true });
});

app.post('/webhooks/anycrawl', async (req, res) => {
  const eventType = req.headers['x-webhook-event'];
  const deliveryId = req.headers['x-webhook-delivery-id'];

  await db.webhookEvents.create({
    deliveryId,
    eventType,
    payload: req.body,
    receivedAt: new Date()
  });

  res.status(200).json({ received: true });
});

curl -X PUT "https://api.anycrawl.dev/v1/webhooks/:webhookId/activate" \
  -H "Authorization: Bearer <your-api-key>"

ngrok http 3000

https://abc123.ngrok.io/webhooks/anycrawl