API

This page demonstrates various ways to integrate with xhr.dev.

Don't see your codebase/stack? Message us!

Disable SSL verification

xhr.dev man-in-the-middle’s (MITM) the requests/responses to get past the bot challenges, so you have to turn off SSL certificate checking.

in node.js, you can do this by setting this environment variable:

$ NODE_TLS_REJECT_UNAUTHORIZED=0 node ...

curl with the --insecure or -k flag:

$ curl -k ...

python requests with verify=False argument:

requests.get('...', verify=False)

Custom Headers and Cookies

Headers you pass in are passed to the target server. All x-xhr-* headers are removed before sent to the origin server.

Control headers

Control headers change the behaviour of how the proxy operates

Required control headers:

• x-xhr-api-key: Your API key

Optional control headers:

• x-xhr-managed-proxy: true - use xhr.dev's managed proxy. default: none (no third-party proxy provided)
• x-xhr-proxy: [url] - bring your own proxy. default: none (no third-party proxy provided)

managed proxy

x-xhr-managed-proxy: true - use xhr.dev's managed proxy

proxy

x-xhr-proxy: [url] Bring your own proxy. Pass in your proxy URL. Will be validated before use. Cannot be combined with x-xhr-managed-proxy.

Node.js

Node.js with Fetch

import { fetch, ProxyAgent } from 'undici';

const proxyUrl = 'https://magic.xhr.dev';

// native fetch requires n@18+
const response = await fetch(url, {
  dispatcher: new ProxyAgent(proxyUrl),
  headers: {
    'x-xhr-api-key': process.env.XHR_API_KEY,
  },
  redirect: 'manual',
});

console.log('Response:', response);

Node.js with Axios (with CookieJar) popular

import axios from 'axios';
import { HttpsProxyAgent } from 'https-proxy-agent';
import { createCookieAgent } from 'http-cookie-agent/http';
import { CookieJar } from 'tough-cookie';

const HttpsProxyCookieAgent = createCookieAgent(HttpsProxyAgent);
const proxyUrl = 'https://magic.xhr.dev';
const jar = new CookieJar();

const httpsProxyCookieAgent = new HttpsProxyCookieAgent(proxyUrl, {
  cookies: { jar },
});

const { data } = await axios.request({
  headers: {
    'x-xhr-api-key': process.env.XHR_API_KEY,
  },
  httpsAgent: httpsProxyCookieAgent,
  url: 'https://app.example.com/login',
  method: 'POST',
});

console.log('Response:', data);

Python popular

import os
import requests

xhr_api_key = os.getenv('XHR_API_KEY')
headers = {
  "x-xhr-api-key": xhr_api_key,
}
proxy_url = 'https://magic.xhr.dev'
proxies = {
  'http': proxy_url,
  'https': proxy_url,
}

session = requests.Session()

response = session.get(
  'https://core.cro.ie',
  headers=headers,
  proxies=proxies,
  verify=False
)

print('Response:', response.text)

Playwright popular

import { chromium } from 'playwright';

const url = 'https://news.ycombinator.com';

const browser = await chromium.launch();
const context = await browser.newContext({
  extraHTTPHeaders: {
    'x-xhr-api-key': xhrApiKey,
  },
  ignoreHTTPSErrors: true,
  proxy: {
    server: proxyUrl,
  },
});
const page = await context.newPage();

try {
  await page.goto(url);

  const title = await page.title();
  const content = await page.content();

  console.log({ content, title });
} finally {
  await browser.close();
}

Puppeteer popular

import puppeteer from 'puppeteer';

const url = 'https://news.ycombinator.com';

const browser = await puppeteer.launch({
  acceptInsecureCerts: true,
  args: [`--proxy-server=${proxyUrl}`], // Set up the proxy
});
const page = await browser.newPage();

try {
  await page.goto(url);

  const title = await page.title();
  const content = await page.content();

  console.log({ content, title });
} finally {
  await browser.close();
}

Go

package main

import (
    "fmt"
    "io/ioutil"
    "log"
    "net/http"
    "net/http/cookiejar"
    "net/url"
    "os"
)

func main() {
    apiKey := os.Getenv("XHR_API_KEY")
    if apiKey == "" {
        log.Fatal("API key is not defined. Please set the 'XHR_API_KEY' environment variable.")
    }

    proxyURL, err := url.Parse("https://magic.xhr.dev")
    if err != nil {
        log.Fatal(err)
    }

    jar, err := cookiejar.New(nil)
    if err != nil {
        log.Fatal(err)
    }

    client := &http.Client{
        Transport: &http.Transport{
            Proxy: http.ProxyURL(proxyURL),
        },
        Jar: jar,
    }

    req, err := http.NewRequest("POST", "https://app.example.com/login", nil)
    if err != nil {
        log.Fatal(err)
    }

    req.Header.Set("x-xhr-api-key", apiKey)

    resp, err := client.Do(req)
    if err != nil {
        log.Fatal(err)
    }
    defer resp.Body.Close()

    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        log.Fatal(err)
    }

    fmt.Println("Response:", string(body))
}

Ruby

require 'net/http'
require 'uri'
require 'http/cookie_jar'

api_key = ENV['XHR_API_KEY']
abort("API key is not defined. Please set the 'XHR_API_KEY' environment variable.") if api_key.nil? || api_key.empty?

proxy_uri = URI.parse('https://magic.xhr.dev')
uri = URI.parse('https://app.example.com/login')

jar = HTTP::CookieJar.new
http = Net::HTTP.new(uri.host, uri.port, proxy_uri.host, proxy_uri.port, nil, nil)
http.use_ssl = true

request = Net::HTTP::Post.new(uri.request_uri)
request['x-xhr-api-key'] = api_key

cookies = jar.cookies(uri)
request['Cookie'] = cookies.map { |c| "#{c.name}=#{c.value}" }.join('; ') unless cookies.empty?

response = http.request(request)

jar.parse(response['Set-Cookie'], uri) if response['Set-Cookie']

puts "Response: #{response.body}"