Documentation

Overview

Learn how to take advantage of GridPanel's APIs

GridPanel has one API you can use:

The Mobile Proxy API
The Scraping API

Below you will find information on how to use and make the most out of these APIs yourself.

Authentication

api_key [string] (default = "") required

All requests, except when explicitly mentioned, to both GridPanel APIs are authenticated using your private API key.

To get access to your API key, create an account and confirm your email address. You can do this here

User Information

GET /api/user

Description

Get information about the user this API key belongs to.

Example

curl 'https://gridpanel.net/api/user?api_key=API_KEY'

import requests

params = {
    'api_key': 'API_KEY',
}

response = requests.get('https://gridpanel.net/api/user', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/user')
params = {
  :api_key => 'API_KEY',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/user?api_key=API_KEY", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/user?api_key=API_KEY');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/user?api_key=API_KEY"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/user?api_key=API_KEY');

Response

    {
  "user": {
    "first_name": "Bob",
    "last_name": "Marley",
    "email": "bob@example.com",
    "gbp_balance_in_pence": 10000
  }
}

  

Proxy API Overview

Our mobile proxy API enables you to: rotate your proxies, get information regarding your usage and your orders in general.

Please note, in order to use the proxy endpoints below, you will need to have an active proxy order on your account.

You can order your GridPanel mobile proxies here.

Rotate your IP address

GET /api/reboot

Description

This endpoint allows you to rotate the IP of your mobile proxy.

Please note that this endpoint does not require your api_key as a parameter.

When you call this endpoint, it will take your proxy down for 20-40 seconds in order for us to get you a new IP address. This is because rotation involves taking the device offline then online again in quick succession. If you have a 5G proxy, it will take your proxy down for around 5-10 seconds as the process is much faster here.

`name` [`type`] (`default_value`)	Description
`token` [`string`] (`required`)	This is your rotation token, you will be given one per order and can find it on your manage order page.

Example

curl 'https://gridpanel.net/api/reboot?token=MsIzNSwwIWE1YjnmMC03Yzk2LTRlY2QtOTVhOC1iNDI1ZWo3MTbzOGi'

import requests

params = {
    'token': 'MsIzNSwwIWE1YjnmMC03Yzk2LTRlY2QtOTVhOC1iNDI1ZWo3MTbzOGi',
}

response = requests.get('https://gridpanel.net/api/reboot', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/reboot')
params = {
  :token => 'MsIzNSwwIWE1YjnmMC03Yzk2LTRlY2QtOTVhOC1iNDI1ZWo3MTbzOGi',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/reboot?token=MsIzNSwwIWE1YjnmMC03Yzk2LTRlY2QtOTVhOC1iNDI1ZWo3MTbzOGi", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/reboot?token=MsIzNSwwIWE1YjnmMC03Yzk2LTRlY2QtOTVhOC1iNDI1ZWo3MTbzOGi');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/reboot?token=MsIzNSwwIWE1YjnmMC03Yzk2LTRlY2QtOTVhOC1iNDI1ZWo3MTbzOGi"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/reboot?token=MsIzNSwwIWE1YjnmMC03Yzk2LTRlY2QtOTVhOC1iNDI1ZWo3MTbzOGi');

Response

In the case of a successful rotation you will receive the following response.

    {
    "success": true
}
  

If you attempt to rotate to frequently, in that you rotate faster than you orders minimum rotation time, you will receive the following response.

    {
    "success": false,
    "reason": "too_frequent"
}
  

Proxy Information

GET /api/proxies

Description

Get all proxy orders in your account

Example

curl 'https://gridpanel.net/api/proxies?api_key=API_KEY'

import requests

params = {
    'api_key': 'API_KEY',
}

response = requests.get('https://gridpanel.net/api/proxies', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/proxies')
params = {
  :api_key => 'API_KEY',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/proxies?api_key=API_KEY", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/proxies?api_key=API_KEY');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/proxies?api_key=API_KEY"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/proxies?api_key=API_KEY');

Response

    {
  "orders": {
    "10123": {
      "port": 10123,
      "status": "active",
      "expiry_date": "2023-03-30T00:00:00Z",
      "plan": "5G Mobile Proxies",
      "duration": "Monthly",
      "auto_renew": true,
      "auto_rotate": false,
      "auto_rotate_seconds": null,
      "last_reboot": "2023-03-04T12:34:56Z",
      "minimum_rotation_time_seconds": 300,
      "gateway_ip": "192.168.1.1",
      "username": "username",
      "password": "password",
      "reboot_url": "https://gridpanel.net/api/reboot?token=1234",
      "reboot_token": "1234",
      "connection_string": "username:password@192.168.1.1:10123",
      "network": "THREE",
      "location": "LONDON",
    },
    "10124": {
      "port": 10124,
      "status": "active",
      "expiry_date": "2023-03-30T00:00:00Z",
      "plan": "4G Mobile Proxies",
      "duration": "Monthly",
      "auto_renew": true,
      "auto_rotate": false,
      "auto_rotate_seconds": null,
      "last_reboot": "2023-03-06T12:00:00Z",
      "minimum_rotation_time_seconds": 300,
      "gateway_ip": "192.168.1.1",
      "username": "username",
      "password": "password",
      "reboot_url": "https://gridpanel.net/api/reboot?token=1235",
      "reboot_token": "1235",
      "connection_string": "username:password@192.168.1.1:10124",
      "network": "EE",
      "location": "LONDON",
    }
  }
}
  

Proxy Statistics

GET /api/statistics

Description

Returns daily statistics (such as bandwidth usage) for one or more proxies, across a date range.

`name` [`type`] (`default_value`)	Description
`ports` [`string`] (`required`)	Proxy port(s) to query for (comma-separated) e.g. 10123,10124
`start` [`string`] (`required`)	Date to query from in YYYY-MM-DD format e.g. 2023-01-01
`end` [`string`] (`required`)	Date to query to in YYYY-MM-DD format e.g. 2023-01-01. Must be within 31 days of start but more than 48 hours before the time of request.

Example

curl 'https://gridpanel.net/api/statistics?api_key=API_KEY&ports=10123,10124&start=2023-03-01&end=2023-03-04'

import requests

response = requests.get(
    'https://gridpanel.net/api/statistics?api_key=API_KEY&ports=10123,10124&start=2023-03-01&end=2023-03-04',
)

require 'net/http'

uri = URI('https://gridpanel.net/api/statistics?api_key=API_KEY&ports=10123,10124&start=2023-03-01&end=2023-03-04')
res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/statistics?api_key=API_KEY&ports=10123,10124&start=2023-03-01&end=2023-03-04", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/statistics?api_key=API_KEY&ports=10123,10124&start=2023-03-01&end=2023-03-04');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/statistics?api_key=API_KEY&ports=10123,10124&start=2023-03-01&end=2023-03-04"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/statistics?api_key=API_KEY&ports=10123,10124&start=2023-03-01&end=2023-03-04');

Response

    {
  "results": {
    "10123": [
      {
        "date": "2023-03-04",
        "used_bytes": 11111111111
      },
      {
        "date": "2023-03-03",
        "used_bytes": 22222222222
      },
      {
        "date": "2023-03-02",
        "used_bytes": 33333333333
      },
      {
        "date": "2023-03-01",
        "used_bytes": 44444444444
      }
    ],
    "10124": [
      {
        "date": "2023-03-04",
        "used_bytes": 55555555555
      },
      {
        "date": "2023-03-03",
        "used_bytes": 66666666666
      },
      {
        "date": "2023-03-02",
        "used_bytes": 77777777777
      },
      {
        "date": "2023-03-01",
        "used_bytes": 88888888888
      }
    ]
  }
}
  

Scraping API Overview

GET /api/scrape

GridPanel's scraping API is the easiest scraping API available to use on the web.

To scrape any web page, you need two things:

Your API key, available here
The encoded web page URL you want to scrape

The following is an example snippet of a simple GET API call to scrape the URL defined in the query string variable YOUR-URL :

curl 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL'

import requests

params = {
    'api_key': 'API_KEY',
    'url': 'YOUR-URL',
}

response = requests.get('https://gridpanel.net/api/scrape', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape')
params = {
  :api_key => 'API_KEY',
  :url => 'YOUR-URL',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL');

The API will then respond with the raw HTML content of the target URL:

    <html>

    <head>
        ...
    </head>

    <body>
        ...
    </body>

</html>
  

Be aware of the maximum 30-second timeout when your code calls the API

Headers and cookies returned by the target website are prefixed with Spb- (for Scraping browser)

Parameters

Here is the list of the different parameters you can use with GridPanel's HTML API.

`name` [`type`] (`default_value`)	Description
`api_key` [`string`] (`required`)	Your API key Get your API key
`url` [`string`] (`required`)	The URL of the page you want to scrape Learn More
`country_code` [`string`] (`""`)	Geolocation, either UK or US Learn More
`extract_rules` [`[stringified JSON]`] (`""`)	Data extraction from CSS selectors Learn More
`js_scenario` [`[stringified JSON]`] (`{}`)	JavaScript scenario to execute Learn More
`json_response` [`bool`] (`false`)	Wrap the response in JSON Learn More
`screenshot` [`boolean`] (`false`)	Return a screenshot of the page you want to scrape Learn More
`timeout` [`integer`] (`14000`)	Timeout for your requests Learn More
`wait` [`integer`] (`0`)	Additional time in ms for JavaScript to render Learn More
`wait_browser` [`string`] (`domcontentloaded`)	Wait until certain browser conditions are met before returning the response Learn More
`wait_for` [`string`] (`""`)	CSS/XPath selector to wait for in the DOM Learn More

URL

url [string] (default = "") required

This parameter is the full URL including the protocol (with http/https ) of the page to extract data from.

You must encode your URL. For example, the + character is encoded to %20 . Consult your programming language documentation for functions that encode URLs.

Alternatively you can encode/decode urls using this tool: https://meyerweb.com/eric/tools/dencoder/

We have provided a number of examples for some languages below:

p = URI::Parser.new
p.escape("YOUR-URL")

sudo apt-get install gridsite-clients
urlencode "YOUR URL"

import urllib.parse
encoded_url = urllib.parse.quote("YOUR URL")

<?php

$url_encoded = urlencode("YOUR URL");

?>

String encoded_url = URLEncoder.encode("YOUR URL", "UTF-8");

package main

import (
"net/url"
)

func main() {
encoded_url := url.QueryEscape("YOUR URL")
}

encoded_url = encodeURIComponent("YOUR URL")

Wait a fixed amount of time

wait [integer] (default = 0)

Most websites take some time to fully render their full content. Using the wait parameter you can tell GridPanel to wait before it returns the fully rendered HTML.

The wait parameter can be a value in milliseconds between 0 and 35000.

Below you can see an example where we are waiting 10000ms (10 seconds) for the page to load before we return the HTML.

curl "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait=10000"

import requests

params = {
    'api_key': 'API_KEY',
    'url': 'YOUR-URL',
    'wait': '10000',
}

response = requests.get('https://gridpanel.net/api/scrape', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape')
params = {
  :api_key => 'API_KEY',
  :url => 'YOUR-URL',
  :wait => '10000',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait=10000", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait=10000');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait=10000"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait=10000');

Wait for selector

wait_for [string] (default = "")

Instead of waiting a fixed amount of time, you might just want to wait for a particular element to appear in the DOM before returning the HTML.

You can utilise the wait_for parameter to pass a CSS/XPath selector to wait for before returning any HTML.

Below you can see an example of us waiting for the div element with a class value of a-loading-div, therefore we can use wait_for=.a-loading-div

curl "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_for=.a-loading-div"

import requests

params = {
    'api_key': 'API_KEY',
    'url': 'YOUR-URL',
    'wait_for': '.a-loading-div',
}

response = requests.get('https://gridpanel.net/api/scrape', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape')
params = {
  :api_key => 'API_KEY',
  :url => 'YOUR-URL',
  :wait_for => '.a-loading-div',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_for=.a-loading-div", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_for=.a-loading-div');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_for=.a-loading-div"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_for=.a-loading-div');

Wait for the browser

wait_browser [string] (default = load)

Instead of waiting for a specific time period, or for an element to appear, you can wait until specific network conditions have been met.

This parameter can take one of 4 different values:

domcontentloaded: Wait until the DOM is loaded
load (default): Wait until the page is fully loaded
networkidle0: Wait until there are no more than 0 network connections for at least 500ms
networkidle2: Wait until there are no more than 2 network connections for at least 500ms

Below you can see an example where we are waiting for the DOM to load by passing wait_browser=domcontentloaded.

curl "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_browser=domcontentloaded"

import requests

params = {
    'api_key': 'API_KEY',
    'url': 'YOUR-URL',
    'wait_browser': 'domcontentloaded',
}

response = requests.get('https://gridpanel.net/api/scrape', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape')
params = {
  :api_key => 'API_KEY',
  :url => 'YOUR-URL',
  :wait_browser => 'domcontentloaded',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_browser=domcontentloaded", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_browser=domcontentloaded');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_browser=domcontentloaded"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&wait_browser=domcontentloaded');

Data Extraction

extract_rules [stringified JSON] (default = "")

If you need to extract any data from the pages you scrape, and don't want to process any HTML on your side, you can take advantage of our extraction rules.

These rules are passed through the extract_rules parameter and use the following syntax:

{ "key_name": "css_or_xpath_selector" }

For example, if you are looking to extract the title, and subtitle of a blog post you will need to use rules that look like the following:

    {
    "title": "h1",
    "subtitle": "#subtitle"
}

  

This will then return the following JSON response:

    {
    "title": "Your blog title",
    "subtitle": "Your blog subtitle"
}

  

It's important to note that these rules are JSON formatted and therefore need to be stringified when you pass them with your requests. You can see an example request here:

curl "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&extract_rules=%7B%22title%22%3A+%22h1%22%2C+%22subtitle%22%3A+%22%23subtitle%22%7D"

import requests

params = {
    'api_key': 'API_KEY',
    'url': 'YOUR-URL',
    'extract_rules': '{"title": "h1", "subtitle": "#subtitle"}',
}

response = requests.get('https://gridpanel.net/api/scrape', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape')
params = {
  :api_key => 'API_KEY',
  :url => 'YOUR-URL',
  :extract_rules => '{"title": "h1", "subtitle": "#subtitle"}',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&extract_rules=%7B%22title%22%3A+%22h1%22%2C+%22subtitle%22%3A+%22%23subtitle%22%7D", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&extract_rules=%7B%22title%22%3A+%22h1%22%2C+%22subtitle%22%3A+%22%23subtitle%22%7D');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&extract_rules=%7B%22title%22%3A+%22h1%22%2C+%22subtitle%22%3A+%22%23subtitle%22%7D"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&extract_rules=%7B%22title%22%3A+%22h1%22%2C+%22subtitle%22%3A+%22%23subtitle%22%7D');

Running JavaScript

js_scenario [stringified JSON] (default = {})

To interact with pages you want to scrape before we return the HTML you can add a JavaScript scenario to your API calls.

For example, to click a button, you can use the following scenario:

    {
    "instructions": [
        { "click": "#buttonId"}
    ]
}

  

This will result with the following API call:

curl "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&js_scenario=%7B%22instructions%22%3A+%5B%7B%22click%22%3A+%22%23buttonId%22%7D%5D%7D"

import requests

params = {
    'api_key': 'API_KEY',
    'url': 'YOUR-URL',
    'js_scenario': '{"instructions": [{"click": "#buttonId"}]}',
}

response = requests.get('https://gridpanel.net/api/scrape', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape')
params = {
  :api_key => 'API_KEY',
  :url => 'YOUR-URL',
  :js_scenario => '{"instructions": [{"click": "#buttonId"}]}',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&js_scenario=%7B%22instructions%22%3A+%5B%7B%22click%22%3A+%22%23buttonId%22%7D%5D%7D", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&js_scenario=%7B%22instructions%22%3A+%5B%7B%22click%22%3A+%22%23buttonId%22%7D%5D%7D');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&js_scenario=%7B%22instructions%22%3A+%5B%7B%22click%22%3A+%22%23buttonId%22%7D%5D%7D"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&js_scenario=%7B%22instructions%22%3A+%5B%7B%22click%22%3A+%22%23buttonId%22%7D%5D%7D');

Again please note that the JavaScript scenario parameter is JSON formatted, which means you need to stringify it when you pass it to your API requests.

Each scenario can have multiple instructions, and they will be executed in the order you send them before returning the HTML of the page.

The supported instructions are as follows:

    {
    "instructions": [
        {"click": "#buttonId"}, // Click a button
        {"wait": 1000}, // Wait a fixed duration in ms
        {"wait_for": #a_div}, // Wait for a specific element
        {"wait_for_and_click": #a_div}, // Wait for an element and click it
        {"scroll_x": 1000}, // Scroll the screen in the horizontal axis, in px
        {"scroll_y": 1000}, // Scroll the screen in the vertical axis, in px
        {"fill": ["#input_1", "value_1"]}, // Fill in some input
        {"evaluate": "console.log(123)", // Evaluate custom javascript code
        {"infinite_scroll": // Scroll the page until end
            {
                "max_count": 0, // Max number to scroll, 0 is forever
                "delay": 1000, // Delay between every scroll in ms
                "end_click": { "selector": "#button_id" } // Click a button when the end of the page is reached
            }
        }
    ]
}

  

If an instruction you need is not supported, you can use the evaluate instruction to run any JavaScript code that you need.

Returning JSON

json_response [boolean] (default = false)

If you want your response format to be in JSON and you want to intercept the responses of XHR/Ajax requests you can use the json_response parameter.

curl "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&json_response=true"

import requests

params = {
    'api_key': 'API_KEY',
    'url': 'YOUR-URL',
    'json_response': 'true',
}

response = requests.get('https://gridpanel.net/api/scrape', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape')
params = {
  :api_key => 'API_KEY',
  :url => 'YOUR-URL',
  :json_response => 'true',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&json_response=true", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&json_response=true');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&json_response=true"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&json_response=true');

You will receive a response that looks like the following:

    
{
  # Headers sent by the server
  "headers": {
    "Date": "Fri, 16 Apr 2021 15:03:54 GMT",
    ...
    "Access-Control-Allow-Credentials": "true"
  },
  # Cost of your request
  "cost": 1,
  # Initial status code of the server
  "initial-status-code": 200,
  # Resolved URL (following redirection)
  "resolved-url": "https://httpbin.org/",
  # Type of the response "html" or "json" or "b64_bytes" for file, image, pdf,...
  "type": "html",
  # Content of the answer. Content will be base 64 encoded if is a file, image, pdf,...
  "body": "<html>... </body>"
  # base 64 encoded screenshot of the page, if screenshot=true is used
  "screenshot": "b0918...aef",
  # Cookies sent back by the server
  "cookies": [
    {
        "name": "cookie_name",
        "value": "cookie_value",
        "domain": "test.com",
        ...
    },
    ...
  ],
  # Results of the JS scenario "evaluate" instructions
  "evaluate_results": [...]
  # Content and source of iframes in the page
  "iframes": [
    {
      "content": "<html>... </body>",
      "src": "https://site.com/iframe"
    },
    ...
  ],
  # XHR / Ajax requests sent by the browser
  "xhr": [
    {
      # URL
      "url": "https://",
      # status code of the server
      "status_code": 200,
      # Method of the request
      "method": "POST",
      # Headers of the XHR / Ajax request
      "headers": {
        "pragma": "no-cache",
        ...
      },
      # Response of the XHR / Ajax request
      "body": "2d,x"
    },
    ...
  ],
  # js_scenario detailed report ( only useful if js_scenario=...)
  "js_scenario_report": {
    "task_executed": 1,
    "task_failure": 0,
    "task_success": 1,
    "tasks": [
        {
            "duration": 3.042,
            "params": 3000,
            "success": true,
            "task": "wait"
        }
    ],
    "total_duration": 3.042
  }, 
  # Metada / Schema data
  "metadata": {
    "microdata": ...,
    "json-ld": ..., 
  }
} 

  

Screenshot

screenshot [boolean] (default = false)

If you require a screenshot of the page that you want to scrape, you can use the screenshot parameter.

Your response will then contain a screenshot key where the value is the screenshot data in base64.

curl "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&screenshot=true" > ./screenshot.png

import requests

params = {
    'api_key': 'API_KEY',
    'url': 'YOUR-URL',
    'screenshot': 'true',
}

response = requests.get('https://gridpanel.net/api/scrape', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape')
params = {
  :api_key => 'API_KEY',
  :url => 'YOUR-URL',
  :screenshot => 'true',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&screenshot=true", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&screenshot=true');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&screenshot=true"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&screenshot=true');

Geolocation

country_code [string] (default = "")

You are also able to chose the proxy location using the country_code parameter.

For example to use proxies from the United Kingdom you need to set country_code=uk

curl "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&country_code=us"

import requests

params = {
    'api_key': 'API_KEY',
    'url': 'YOUR-URL',
    'country_code': 'us',
}

response = requests.get('https://gridpanel.net/api/scrape', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape')
params = {
  :api_key => 'API_KEY',
  :url => 'YOUR-URL',
  :country_code => 'us',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&country_code=us", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&country_code=us');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&country_code=us"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape?api_key=API_KEY&url=YOUR-URL&country_code=us');

Track your usage

GET /api/scrape/usage

Description

Allows you to programmatically monitor scrape consumption.

Results are available in real time, and will not increase your request concurrency and will not cost scrapes.

Example

curl 'https://gridpanel.net/api/scrape/usage?api_key=API_KEY'

import requests

params = {
    'api_key': 'API_KEY',
}

response = requests.get('https://gridpanel.net/api/scrape/usage', params=params)

require 'net/http'

uri = URI('https://gridpanel.net/api/scrape/usage')
params = {
  :api_key => 'API_KEY',
}
uri.query = URI.encode_www_form(params)

res = Net::HTTP.get_response(uri)

package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
)

func main() {
	client := &http.Client{}
	req, err := http.NewRequest("GET", "https://gridpanel.net/api/scrape/usage?api_key=API_KEY", nil)
	if err != nil {
		log.Fatal(err)
	}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()
	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%s
", bodyText)
}

<?php
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, 'https://gridpanel.net/api/scrape/usage?api_key=API_KEY');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');

$response = curl_exec($ch);

curl_close($ch);

import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;

HttpClient client = HttpClient.newHttpClient();

HttpRequest request = HttpRequest.newBuilder()
    .uri(URI.create("https://gridpanel.net/api/scrape/usage?api_key=API_KEY"))
    .GET()
    .build();

HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

fetch('https://gridpanel.net/api/scrape/usage?api_key=API_KEY');

Response

    {
  "result": {
    "max_api_scrapes": 100000,
    "used_api_scrapes": 301,
    "renewal_date": "Free Trial"
}
  

Scrape costs

Every GridPanel plan will provide you a certain number of API scrapes per month.

We want to keep our pricing as simple and as transparent as possible.

We only charge you for successful scrapes (2XX,4XX HTTP responses), and each scrape costs the exact same. No surprises.

Dedicated 4G

Dedicated 5G

Social Media

Datacenter

Residential

Choose your language below

Overview

Learn how to take advantage of GridPanel's APIs

Authentication

User Information

Description

Example

Response

Proxy API Overview

Rotate your IP address

Description

Example

Response

Proxy Information

Description

Example

Response

Proxy Statistics

Description

Example

Response

Scraping API Overview

Parameters

URL

Wait a fixed amount of time

Wait for selector

Wait for the browser

Data Extraction

Running JavaScript

Returning JSON

Screenshot

Geolocation

Track your usage

Description

Example

Response

Scrape costs