Crawl the web. Every page, clean Markdown.

import ContextDev from 'context.dev';

const client = new ContextDev({ apiKey: 'YOUR_API_KEY' });

const batch = await client.batch.submit({
  input: {
    mode: 'crawl',
    data: {
      format: 'markdown',
      source: {
        type: 'start_url',
        url: 'https://acme.com/docs',
        controls: {
          maxUrls: 2500,
          maxDepth: 3,
          regex: '^https://acme\\.com/docs/',
        },
      },
      options: { useMainContentOnly: true },
    },
  },
});

// Poll every 10-30 seconds, or receive a batch webhook instead
let status: string = batch.status;
while (status === 'queued' || status === 'running') {
  await new Promise((resolve) => setTimeout(resolve, 15_000));
  ({ status } = await client.batch.retrieve(batch.id));
}

let cursor: string | undefined;
do {
  const results = await client.batch.getResults(batch.id, {
    limit: 100,
    cursor,
  });
  for (const page of results.data ?? []) {
    if (page.status === 'ok') console.log(page.url, page.markdown);
  }
  cursor = (results.has_more && results.next_cursor) || undefined;
} while (cursor);

import time

from context.dev import ContextDev

client = ContextDev(api_key="YOUR_API_KEY")

batch = client.batch.submit(
    input={
        "mode": "crawl",
        "data": {
            "format": "markdown",
            "source": {
                "type": "start_url",
                "url": "https://acme.com/docs",
                "controls": {
                    "max_urls": 2500,
                    "max_depth": 3,
                    "regex": r"^https://acme\.com/docs/",
                },
            },
            "options": {"use_main_content_only": True},
        },
    },
)

# Poll every 10-30 seconds, or receive a batch webhook instead
status = batch.status
while status in ("queued", "running"):
    time.sleep(15)
    status = client.batch.retrieve(batch.id).status

results = client.batch.get_results(batch.id, limit=100)
while True:
    for page in results.data or []:
        if page.status == "ok":
            print(page.url, page.markdown)
    if not results.has_more:
        break
    results = client.batch.get_results(
        batch.id, limit=100, cursor=results.next_cursor
    )

require "context_dev"

client = ContextDev::Client.new(api_key: "YOUR_API_KEY")

batch = client.batch.submit(
  input: {
    mode: :crawl,
    data: {
      format: :markdown,
      source: {
        type: :start_url,
        url: "https://acme.com/docs",
        controls: {
          max_urls: 2500,
          max_depth: 3,
          regex: "^https://acme\\.com/docs/"
        }
      },
      options: { use_main_content_only: true }
    }
  }
)

# Poll every 10-30 seconds, or receive a batch webhook instead
status = batch.status
while %i[queued running].include?(status)
  sleep 15
  status = client.batch.retrieve(batch.id).status
end

results = client.batch.get_results(batch.id, limit: 100)
loop do
  results.data&.each do |page|
    puts "#{page.url} #{page.markdown}" if page.status == :ok
  end
  break unless results.has_more
  results = client.batch.get_results(
    batch.id, limit: 100, cursor: results.next_cursor
  )
end

<?php

require_once 'vendor/autoload.php';

use ContextDev\Client;

$client = new Client(apiKey: 'YOUR_API_KEY');

$batch = $client->batch->submit(
    input: [
        'mode' => 'crawl',
        'data' => [
            'format' => 'markdown',
            'source' => [
                'type' => 'start_url',
                'url' => 'https://acme.com/docs',
                'controls' => [
                    'maxURLs' => 2500,
                    'maxDepth' => 3,
                    'regex' => '^https://acme\.com/docs/',
                ],
            ],
            'options' => ['useMainContentOnly' => true],
        ],
    ],
);

// Poll every 10-30 seconds, or receive a batch webhook instead
$status = $batch->status;
while (in_array($status, ['queued', 'running'], true)) {
    sleep(15);
    $status = $client->batch->retrieve($batch->id)->status;
}

$cursor = null;
do {
    $results = $client->batch->getResults(
        $batch->id,
        cursor: $cursor,
        limit: 100,
    );
    foreach ($results->data ?? [] as $page) {
        if ($page->status === 'ok') {
            echo $page->url, PHP_EOL, $page->markdown, PHP_EOL;
        }
    }
    $cursor = $results->hasMore ? $results->nextCursor : null;
} while ($cursor !== null);

package main

import (
	"context"
	"fmt"
	"time"

	contextdev "github.com/context-dot-dev/context-go-sdk/v2"
	"github.com/context-dot-dev/context-go-sdk/v2/option"
)

func main() {
	client := contextdev.NewClient(
		option.WithAPIKey("YOUR_API_KEY"),
	)

	batch, err := client.Batch.Submit(context.TODO(), contextdev.BatchSubmitParams{
		Input: contextdev.BatchSubmitParamsInputUnion{
			OfCrawl: &contextdev.BatchSubmitParamsInputCrawl{
				Data: contextdev.BatchSubmitParamsInputCrawlDataUnion{
					OfMarkdown: &contextdev.BatchSubmitParamsInputCrawlDataMarkdown{
						Source: contextdev.BatchSubmitParamsInputCrawlDataMarkdownSourceUnion{
							OfStartURL: &contextdev.BatchSubmitParamsInputCrawlDataMarkdownSourceStartURL{
								URL: "https://acme.com/docs",
								Controls: contextdev.BatchSubmitParamsInputCrawlDataMarkdownSourceStartURLControls{
									MaxURLs:  contextdev.Int(2500),
									MaxDepth: contextdev.Int(3),
									Regex:    contextdev.String("^https://acme\\.com/docs/"),
								},
							},
						},
						Options: contextdev.BatchSubmitParamsInputCrawlDataMarkdownOptions{
							UseMainContentOnly: contextdev.Bool(true),
						},
					},
				},
			},
		},
	})
	if err != nil {
		panic(err)
	}

	// Poll every 10-30 seconds, or receive a batch webhook instead
	status := string(batch.Status)
	for status == "queued" || status == "running" {
		time.Sleep(15 * time.Second)
		current, err := client.Batch.Get(context.TODO(), batch.ID)
		if err != nil {
			panic(err)
		}
		status = string(current.Status)
	}

	params := contextdev.BatchGetResultsParams{Limit: contextdev.Int(100)}
	for {
		results, err := client.Batch.GetResults(context.TODO(), batch.ID, params)
		if err != nil {
			panic(err)
		}
		for _, page := range results.Data {
			if page.Status == "ok" {
				fmt.Println(page.URL, page.Markdown)
			}
		}
		if !results.HasMore {
			break
		}
		params.Cursor = contextdev.String(results.NextCursor)
	}
}

curl -X POST "https://api.context.dev/v1/batch/submit" \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -d '{
  "input": {
    "mode": "crawl",
    "data": {
      "format": "markdown",
      "source": {
        "type": "start_url",
        "url": "https://acme.com/docs",
        "controls": {
          "maxUrls": 2500,
          "maxDepth": 3,
          "regex": "^https://acme\\.com/docs/"
        }
      },
      "options": { "useMainContentOnly": true }
    }
  }
}'

# Poll with the returned id until status is "completed"
curl "https://api.context.dev/v1/batch/BATCH_ID" \
  -H "Authorization: Bearer YOUR_API_KEY"

curl "https://api.context.dev/v1/batch/BATCH_ID/results?limit=100" \
  -H "Authorization: Bearer YOUR_API_KEY"