For the complete documentation index, see llms.txt.

Robots.txt Generator API

webmaster 1 credit / call v2026-04-22

Generate a robots.txt from allow/disallow paths, extra user-agent groups, and sitemaps, or validate an existing file for errors and crawler warnings.

One POST adds Robots.txt Generator to your app, site, workflow, or agent — formula, validation, edge cases, and docs already handled.

prefer a UI? Open the Robots.txt Generator on miniwebtool.com →

Start free — get a key → 1,000 credits · no card · 30 seconds

Endpoint

POST · 1cr
POST https://api.miniwebtool.com/v1/tools/robots-txt-generator/run

Request body

Field Type Req.
mode
generate | validate
str
site_url
site base URL, used to absolutise sitemap paths
str
default_user_agent
user agent for the primary group
str
allow_paths
Allow paths, one per line
str
disallow_paths
Disallow paths, one per line
str
sitemap_urls
sitemap URLs or paths, one per line
str
crawl_delay
optional Crawl-delay in seconds
str
host
optional Host directive
str
clean_param
optional Clean-param directives, one per line
str
comment_note
optional comment lines placed at the top
str
custom_agent_blocks
extra groups, separated by blank lines; first line is the user agent
str
raw_robots
existing robots.txt to check when mode=validate
str

Cost & access

  • 1 credit per successful call — failed calls refund.
  • Starts on: Free.
  • Max payload: 524288 bytes.
  • Privacy mode: no_body_logs
  • Available on: free, starter, pro, business, scale

Response envelope

{
  "request_id": "01K...",
  "tool": "robots-txt-generator",
  "tool_version": "2026-04-22",
  "credits_used": 1,
  "result": { ... }
}

Try it

Stored only in this browser tab. Playground test key is prefilled and only works on this page. Get a live key →
generate | validate
site base URL, used to absolutise sitemap paths
user agent for the primary group
Allow paths, one per line
Disallow paths, one per line
sitemap URLs or paths, one per line
optional Crawl-delay in seconds
optional Host directive
optional Clean-param directives, one per line
optional comment lines placed at the top
extra groups, separated by blank lines; first line is the user agent
existing robots.txt to check when mode=validate

Get your own key

1,000 free calls/mo · no card · key arrives by email

One verification click, then your key + a ready-to-run request. No card, no spam.

Code examples

curl -X POST https://api.miniwebtool.com/v1/tools/robots-txt-generator/run \
  -H 'Authorization: Bearer mwt_live_YOUR_KEY' \
  -H 'Content-Type: application/json' \
  -d '{"mode":"generate","site_url":"https://example.com","disallow_paths":"/admin/\n/private/","sitemap_urls":"/sitemap.xml"}'
import requests

resp = requests.post(
    'https://api.miniwebtool.com/v1/tools/robots-txt-generator/run',
    headers={'Authorization': 'Bearer mwt_live_YOUR_KEY'},
    json={'mode': 'generate', 'site_url': 'https://example.com', 'disallow_paths': '/admin/\n/private/', 'sitemap_urls': '/sitemap.xml'},
)
resp.raise_for_status()
data = resp.json()
print(data)
const resp = await fetch(
  'https://api.miniwebtool.com/v1/tools/robots-txt-generator/run',
  {
    method: 'POST',
    headers: {
      'Authorization': 'Bearer mwt_live_YOUR_KEY',
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({"mode":"generate","site_url":"https://example.com","disallow_paths":"/admin/\n/private/","sitemap_urls":"/sitemap.xml"}),
  }
);
const data = await resp.json();
console.log(data);
<?php
$ch = curl_init('https://api.miniwebtool.com/v1/tools/robots-txt-generator/run');
curl_setopt_array($ch, [
    CURLOPT_POST => true,
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_HTTPHEADER => [
        'Authorization: Bearer mwt_live_YOUR_KEY',
        'Content-Type: application/json',
    ],
    CURLOPT_POSTFIELDS => json_encode([
    'mode' => 'generate',
    'site_url' => 'https://example.com',
    'disallow_paths' => '/admin/
/private/',
    'sitemap_urls' => '/sitemap.xml'
]),
]);
$resp = curl_exec($ch);
curl_close($ch);
print_r(json_decode($resp, true));
require 'net/http'
require 'json'
require 'uri'

uri = URI('https://api.miniwebtool.com/v1/tools/robots-txt-generator/run')
http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = (uri.scheme == 'https')

req = Net::HTTP::Post.new(uri)
req['Authorization'] = 'Bearer mwt_live_YOUR_KEY'
req['Content-Type'] = 'application/json'
req.body = {
  'mode' => 'generate',
  'site_url' => 'https://example.com',
  'disallow_paths' => '/admin/
/private/',
  'sitemap_urls' => '/sitemap.xml'
}.to_json

resp = http.request(req)
puts JSON.parse(resp.body).inspect
package main

import (
  "bytes"
  "encoding/json"
  "fmt"
  "io"
  "net/http"
)

func main() {
  body, _ := json.Marshal(map[string]interface{}{
    "mode": "generate",
    "site_url": "https://example.com",
    "disallow_paths": "/admin/
/private/",
    "sitemap_urls": "/sitemap.xml",
  })
  req, _ := http.NewRequest("POST", "https://api.miniwebtool.com/v1/tools/robots-txt-generator/run", bytes.NewReader(body))
  req.Header.Set("Authorization", "Bearer mwt_live_YOUR_KEY")
  req.Header.Set("Content-Type", "application/json")

  resp, err := http.DefaultClient.Do(req)
  if err != nil { panic(err) }
  defer resp.Body.Close()
  out, _ := io.ReadAll(resp.Body)
  fmt.Println(string(out))
}
import java.net.URI;
import java.net.http.*;

var client = HttpClient.newHttpClient();
var req = HttpRequest.newBuilder()
    .uri(URI.create("https://api.miniwebtool.com/v1/tools/robots-txt-generator/run"))
    .header("Authorization", "Bearer mwt_live_YOUR_KEY")
    .header("Content-Type", "application/json")
    .POST(HttpRequest.BodyPublishers.ofString("{\"mode\":\"generate\",\"site_url\":\"https://example.com\",\"disallow_paths\":\"/admin/\\n/private/\",\"sitemap_urls\":\"/sitemap.xml\"}"))
    .build();

var resp = client.send(req, HttpResponse.BodyHandlers.ofString());
System.out.println(resp.body());
how-to call robots-txt-generator
  1. Get an API key
    Get an API key. Sign up free for a bearer token. 1,000 credits/month, no card.
  2. Assemble the JSON body
    Required fields: . See the Request body table above for the full schema.
  3. POST to https://api.miniwebtool.com/v1/tools/robots-txt-generator/run
    Headers: Authorization: Bearer <key> and Content-Type: application/json. Copy-paste snippets in 7 languages are under Code examples.
  4. Parse the response
    Envelope: {request_id, tool, tool_version, credits_used, result}. The typed output lives in result.
faq frequently asked
What does the Robots.txt Generator API do?+

Generate a robots.txt from allow/disallow paths, extra user-agent groups, and sitemaps, or validate an existing file for errors and crawler warnings. Use it to add this utility without rebuilding formulas, validation, examples, and documentation.

How much does one call to Robots.txt Generator cost?+

1 credit per successful call. Failed calls (validation errors, 5xx) don't bill.

What parameters does the Robots.txt Generator API require?+

Required fields: (none). Full schema at /v1/openapi.json.

Is the Robots.txt Generator API deterministic?+

Yes — same input, same output, forever. Tool version 2026-04-22; output shape is stable within a major version.

Can an AI agent call the Robots.txt Generator API?+

Yes. Connect to https://api.miniwebtool.com/v1/mcp over the Model Context Protocol. The agent discovers `robots-txt-generator` with its JSON Schema automatically, so the model can plan while MiniWebtool handles the exact tool result.

What format does the API return?+

JSON with a stable envelope: `{request_id, tool, tool_version, credits_used, result}`. Errors are RFC 7807 `application/problem+json`.

mcp use from an AI agent

Every endpoint on this site is also exposed via the Model Context Protocol at https://api.miniwebtool.com/v1/mcp. Claude, Cursor, and any MCP-capable agent can discover this tool (robots-txt-generator) and its JSON Schema automatically — no client codegen.

# List tools the agent can call
curl -s https://api.miniwebtool.com/v1/mcp \
  -H 'Content-Type: application/json' \
  -d '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' \
 | jq '.result.tools[] | select(.name=="robots-txt-generator")'
see-also related webmaster tools