Eddie.surf API

const response = await fetch('https://api.eddie.surf/crawl', {
  method: 'POST',
  headers: { 
    'X-API-Key': 'your-api-key-here',
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    urls: ['https://data-surfer.com'],
    context: {purpose: 'Company research'},
    json: {
      company_name: {
        type: 'string',
        description: 'Company name'
      }
    }
  })
});

const data = await response.json();
console.log(data);

import requests

response = requests.post('https://api.eddie.surf/crawl', 
    headers={'X-API-Key': 'your-api-key-here'},
    json={
    'urls': ['https://data-surfer.com'],
    'context': {'purpose': 'Company research'},
    'json': {
        'company_name': {
            'type': 'string',
            'description': 'Company name'
        }
    }
})

data = response.json()
print(data)

require 'net/http'
require 'json'
require 'uri'

uri = URI('https://api.eddie.surf/crawl')
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Post.new(uri)
request['X-API-Key'] = 'your-api-key-here'
request['Content-Type'] = 'application/json'
request.body = {
  urls: ['https://data-surfer.com'],
  context: { purpose: 'Company research' },
  json: {
    company_name: {
      type: 'string',
      description: 'Company name'
    }
  }
}.to_json

response = http.request(request)
data = JSON.parse(response.body)
puts data

<?php
$url = 'https://api.eddie.surf/crawl';
$data = [
    'urls' => ['https://data-surfer.com'],
    'context' => ['purpose' => 'Company research'],
    'json' => [
        'company_name' => [
            'type' => 'string',
            'description' => 'Company name'
        ]
    ]
];

$options = [
    'http' => [
        'header' => "X-API-Key: your-api-key-here\r\n" .
                   "Content-Type: application/json\r\n",
        'method' => 'POST',
        'content' => json_encode($data)
    ]
];

$context = stream_context_create($options);
$response = file_get_contents($url, false, $context);
$result = json_decode($response, true);
print_r($result);
?>

package main

import (
    "bytes"
    "encoding/json"
    "fmt"
    "net/http"
)

func main() {
    url := "https://api.eddie.surf/crawl"
    
    payload := map[string]interface{}{
        "urls":    []string{"https://data-surfer.com"},
        "context": map[string]string{"purpose": "Company research"},
        "json": map[string]interface{}{
            "company_name": map[string]string{
                "type":        "string",
                "description": "Company name",
            },
        },
    }
    
    jsonData, _ := json.Marshal(payload)
    
    req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonData))
    req.Header.Set("X-API-Key", "your-api-key-here")
    req.Header.Set("Content-Type", "application/json")
    
    client := &http.Client{}
    resp, err := client.Do(req)
    if err != nil {
        panic(err)
    }
    defer resp.Body.Close()
    
    var result map[string]interface{}
    json.NewDecoder(resp.Body).Decode(&result)
    fmt.Println(result)
}

{
  "status": "completed",
  "job_id": 123,
  "total_sites": 3,
  "completed_sites": 2,
  "failed_sites": 1,
  "processing_sites": 0,
  "credits_remaining": 835,
  "credits_used": 19.5,
  "sites": [
    {
      "site_id": 456,
      "url": "https://data-surfer.com",
      "status": "completed",
      "completed_pages": 5,
      "failed_pages": 0,
      "pending_pages": 0,
      "results": {
        "company_name": {
          "value": "Data Surfer Inc.",
          "confidence": 5,
          "whereFound": "Found at [1]\\n\\nSources:\\n[1] https://data-surfer.com/"
        }
      },
      "created_at": "2025-01-20T10:30:00Z",
      "updated_at": "2025-01-20T10:35:00Z"
    },
    {
      "site_id": 457,
      "url": "https://eddie.surf",
      "status": "completed",
      "completed_pages": 4,
      "failed_pages": 0,
      "pending_pages": 0,
      "results": {
        "company_name": {
          "value": "Eddie.surf Inc.",
          "confidence": 5,
          "whereFound": "Found at [1]\\n\\nSources:\\n[1] https://eddie.surf/"
        }
      },
      "created_at": "2025-01-20T10:30:00Z",
      "updated_at": "2025-01-20T10:36:00Z"
    },
    {
      "site_id": 458,
      "url": "https://broken-site.com",
      "status": "failed",
      "completed_pages": 0,
      "failed_pages": 3,
      "pending_pages": 0,
      "message": "All pages failed during scraping",
      "created_at": "2025-01-20T10:30:00Z",
      "updated_at": "2025-01-20T10:31:00Z"
    }
  ],
  "created_at": "2025-01-20T10:30:00Z",
  "updated_at": "2025-01-20T10:36:00Z"
}

{
  "status": "failed",
  "job_id": 124,
  "total_sites": 1,
  "completed_sites": 0,
  "failed_sites": 1,
  "processing_sites": 0,
  "credits_remaining": 851,
  "credits_used": 4.5,
  "sites": [
    {
      "site_id": 459,
      "url": "https://broken-site.com",
      "status": "failed",
      "completed_pages": 0,
      "failed_pages": 3,
      "pending_pages": 0,
      "created_at": "2025-01-20T11:00:00Z",
      "updated_at": "2025-01-20T11:01:00Z"
    }
  ],
  "created_at": "2025-01-20T11:00:00Z",
  "updated_at": "2025-01-20T11:01:00Z"
}

// callback_mode: "multi" → 3 separate callbacks as each site completes
// Matches GET /crawl/{job_id}/{site_id} response format
{
  "status": "completed",
  "job_id": 123,
  "site_id": 456,
  "url": "https://data-surfer.com",
  "completed_pages": 10,
  "failed_pages": 5,
  "pending_pages": 0,
  "progress": 67,
  "results": {
    "company_name": {
      "value": "Data Surfer Inc.",
      "confidence": 5,
      "whereFound": "Found at [1]\\n\\nSources:\\n[1] https://data-surfer.com/"
    }
  },
  "message": "Data extracted successfully with 5 pages failed to scrape",
  "created_at": "2025-01-20T10:30:00Z",
  "updated_at": "2025-01-20T10:35:00Z",
  "credits_remaining": 837,
  "credits_used": 18.0
}

import time
import requests

def poll_queue(job_id):
    complete = False
    start_time = time.time()
    
    while not complete:
        response = requests.get(f'https://api.eddie.surf/crawl/{job_id}', 
                               headers={'X-API-Key': 'your-api-key-here'})
        data = response.json()
        
        if data['status'] == 'completed':
            print(f"✅ Complete! Processing {data['total_sites']} sites")
            return data.get('sites', data.get('results'))
        
        if data['status'] == 'failed':
            print(f"❌ Failed: {data['message']}")
            return None
        
        print(f"🔄 Progress: {data.get('progress', 0)}%")
        
        # Dynamic polling interval
        elapsed = time.time() - start_time
        if elapsed < 120:  # First 2 minutes
            wait_time = 10
        elif elapsed < 420:  # Next 5 minutes
            wait_time = 30
        else:  # After that
            wait_time = 60
        
        time.sleep(wait_time)

require 'net/http'
require 'json'
require 'uri'

def poll_crawl_job(job_id)
  start_time = Time.now
  
  loop do
    begin
      uri = URI("https://api.eddie.surf/crawl/#{job_id}")
      http = Net::HTTP.new(uri.host, uri.port)
      http.use_ssl = true
      
      request = Net::HTTP::Get.new(uri)
      request['X-API-Key'] = 'your-api-key-here'
      request['Content-Type'] = 'application/json'
      
      response = http.request(request)
      raise "HTTP #{response.code}: #{response.message}" unless response.is_a?(Net::HTTPSuccess)
      
      data = JSON.parse(response.body)
      
      case data['status']
      when 'completed'
        completed = data['completed_sites'] || 0
        total = data['total_sites'] || 0
        puts "✅ Job completed! #{completed}/#{total} sites successful"
        return data['sites'] || []
      when 'failed'
        puts "❌ Job failed: #{data['message'] || 'Unknown error'}"
        return nil
      else
        # Processing status
        progress = data['progress'] || 0
        completed = data['completed_sites'] || 0
        total = data['total_sites'] || 0
        puts "🔄 Progress: #{progress}% (#{completed}/#{total} sites)"
      end
      
      # Dynamic polling interval
      elapsed = Time.now - start_time
      wait_time = if elapsed < 120        # First 2 minutes
                    10
                  elsif elapsed < 420     # Next 5 minutes
                    30
                  else                    # After that
                    60
                  end
      
      sleep(wait_time)
      
    rescue => e
      puts "Polling error: #{e}"
      sleep(10)  # Wait 10s on error
    end
  end
end

# Usage
sites = poll_crawl_job(123)
puts "Results: #{sites}" if sites

<?php
function pollCrawlJob($jobId) {
    $startTime = time();
    
    while (true) {
        try {
            $url = "https://api.eddie.surf/crawl/$jobId";
            $context = stream_context_create([
                'http' => [
                    'header' => "X-API-Key: your-api-key-here\r\n",
                    'method' => 'GET'
                ]
            ]);
            
            $response = file_get_contents($url, false, $context);
            if ($response === false) {
                throw new Exception('HTTP request failed');
            }
            
            $data = json_decode($response, true);
            if ($data === null) {
                throw new Exception('Invalid JSON response');
            }
            
            switch ($data['status']) {
                case 'completed':
                    $completed = $data['completed_sites'] ?? 0;
                    $total = $data['total_sites'] ?? 0;
                    echo "✅ Job completed! $completed/$total sites successful\n";
                    return $data['sites'] ?? [];
                    
                case 'failed':
                    $message = $data['message'] ?? 'Unknown error';
                    echo "❌ Job failed: $message\n";
                    return null;
                    
                default:
                    // Processing status
                    $progress = $data['progress'] ?? 0;
                    $completed = $data['completed_sites'] ?? 0;
                    $total = $data['total_sites'] ?? 0;
                    echo "🔄 Progress: {$progress}% ($completed/$total sites)\n";
            }
            
            // Dynamic polling interval
            $elapsed = time() - $startTime;
            if ($elapsed < 120) {        // First 2 minutes
                $waitTime = 10;
            } elseif ($elapsed < 420) {  // Next 5 minutes
                $waitTime = 30;
            } else {                     // After that
                $waitTime = 60;
            }
            
            sleep($waitTime);
            
        } catch (Exception $e) {
            echo "Polling error: " . $e->getMessage() . "\n";
            sleep(10);  // Wait 10s on error
        }
    }
}

// Usage
$sites = pollCrawlJob(123);
if ($sites) {
    print_r($sites);
}
?>

package main

import (
    "encoding/json"
    "fmt"
    "net/http"
    "time"
)

type JobStatus struct {
    Status         string `json:"status"`
    JobID          int    `json:"job_id"`
    TotalSites     int    `json:"total_sites"`
    CompletedSites int    `json:"completed_sites"`
    Progress       int    `json:"progress"`
    Message        string `json:"message"`
    Sites          []interface{} `json:"sites"`
}

func pollCrawlJob(jobID int) ([]interface{}, error) {
    startTime := time.Now()
    
    for {
        url := fmt.Sprintf("https://api.eddie.surf/crawl/%d", jobID)
        
        client := &http.Client{Timeout: 30 * time.Second}
        req, err := http.NewRequest("GET", url, nil)
        req.Header.Set("X-API-Key", "your-api-key-here")
        if err != nil {
            return nil, err
        }
        req.Header.Set("Content-Type", "application/json")
        
        resp, err := client.Do(req)
        if err != nil {
            fmt.Printf("Polling error: %v\n", err)
            time.Sleep(10 * time.Second)
            continue
        }
        defer resp.Body.Close()
        
        var data JobStatus
        if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
            fmt.Printf("JSON decode error: %v\n", err)
            time.Sleep(10 * time.Second)
            continue
        }
        
        switch data.Status {
        case "completed":
            fmt.Printf("✅ Job completed! %d/%d sites successful\n", 
                data.CompletedSites, data.TotalSites)
            return data.Sites, nil
            
        case "failed":
            fmt.Printf("❌ Job failed: %s\n", data.Message)
            return nil, fmt.Errorf("job failed: %s", data.Message)
            
        default:
            // Processing status
            fmt.Printf("🔄 Progress: %d%% (%d/%d sites)\n", 
                data.Progress, data.CompletedSites, data.TotalSites)
        }
        
        // Dynamic polling interval
        elapsed := time.Since(startTime)
        var waitTime time.Duration
        if elapsed < 2*time.Minute {        // First 2 minutes
            waitTime = 10 * time.Second
        } else if elapsed < 7*time.Minute { // Next 5 minutes
            waitTime = 30 * time.Second
        } else {                            // After that
            waitTime = 60 * time.Second
        }
        
        time.Sleep(waitTime)
    }
}

func main() {
    sites, err := pollCrawlJob(123)
    if err != nil {
        fmt.Printf("Error: %v\n", err)
        return
    }
    
    fmt.Printf("Results: %+v\n", sites)
}

#!/bin/bash

poll_queue() {
    local job_id=$1
    local start_time=$(date +%s)
    
    while true; do
        response=$(curl -s -H "X-API-Key: your-api-key-here" "https://api.eddie.surf/crawl/${job_id}")
        status=$(echo $response | jq -r '.status')
        
        if [ "$status" = "completed" ]; then
            total=$(echo $response | jq -r '.total_sites')
            echo "✅ Complete! Processing ${total} sites"
            echo $response | jq '.sites // .results'
            break
        fi
        
        if [ "$status" = "failed" ]; then
            message=$(echo $response | jq -r '.message')
            echo "❌ Failed: ${message}"
            break
        fi
        
        progress=$(echo $response | jq -r '.progress // 0')
        echo "🔄 Progress: ${progress}%"
        
        # Dynamic polling interval
        current_time=$(date +%s)
        elapsed=$((current_time - start_time))
        
        if [ $elapsed -lt 120 ]; then
            sleep 10
        elif [ $elapsed -lt 420 ]; then
            sleep 30
        else
            sleep 60
        fi
    done
}

poll_queue "YOUR_JOB_ID"