guest-tools/intengine.py

331 lines
16 KiB
Python

"""Generic integration engine (Mike, 2026-11-13: "it will be thousands of integrations").
One integration = one declarative spec (JSON). This module turns (spec, command, params, credential) into an HTTP call.
The same file is used by osmike-runs (server-side tests, trials) and by the `osmike-int` CLI in every agent.
Stdlib only. Specs are data, never code.
Spec:
{
"slug": "elevenlabs", "name": "ElevenLabs", "version": 1,
"base_url": "https://api.elevenlabs.io",
"allowed_hosts": ["api.elevenlabs.io"], # optional; defaults to base_url's host
"auth": {"type": "api_key", "in": "header", "name": "xi-api-key", "field": "api_key"}
| {"type": "bearer", "field": "token"}
| {"type": "basic", "username_field": "username", "password_field": "password"}
| {"type": "none"}
| {"type": "oauth2", "authorize_url": "...", "token_url": "...", "scopes": ["..."], "scope_separator": " ",
"pkce": true, "client_auth": "body" | "basic", "extra_authorize_params": {}},
"credentials": [{"name": "api_key", "label": "API key"}], # what the person gives (oauth2: client_id, client_secret)
"setup": "Where to get the key / how to create the OAuth app. {redirect_uri} is filled in.",
"headers": {"Accept": "application/json"}, # optional fixed headers
"test": {"command": "me", "params": {}}, # read-only call proving the credential works
"commands": {
"voices": {"summary": "List voices", "method": "GET", "path": "/v1/voices",
"params": {"page_size": {"in": "query", "type": "integer", "description": "..."}}},
"speak": {"summary": "Text to speech", "method": "POST", "path": "/v1/text-to-speech/{voice_id}", "write": false,
"params": {"voice_id": {"in": "path", "required": true}, "text": {"in": "body", "required": true}},
"body_type": "json", "output": {"type": "file", "ext": "mp3"}}
}
}
"write" defaults to true for every method except GET/HEAD; a write command needs the connection's "allow writing".
"""
import base64
import json
import re
import urllib.error
import urllib.parse
import urllib.request
AUTH_TYPES = ('api_key', 'bearer', 'basic', 'none', 'oauth2')
METHODS = ('GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE')
PARAM_IN = ('query', 'path', 'body', 'header')
PARAM_TYPES = ('string', 'integer', 'number', 'boolean', 'json', 'array')
SLUG_RE = re.compile(r'[a-z0-9]+(?:-[a-z0-9]+)*')
NAME_RE = re.compile(r'[A-Za-z0-9_.-]{1,80}')
MAX_BYTES = 25 * 1024 * 1024
class SpecError(ValueError):
pass
class CallError(ValueError):
pass
def host_of(url):
return (urllib.parse.urlsplit(url).hostname or '').lower()
def validate(spec):
"""List of problems (empty = valid). Strict on the shape so a bad spec never reaches an agent."""
errors = []
if not isinstance(spec, dict):
return ['spec must be an object']
slug = spec.get('slug')
if not isinstance(slug, str) or not SLUG_RE.fullmatch(slug) or len(slug) > 60:
errors.append('slug: lowercase letters, digits and dashes')
if not isinstance(spec.get('name'), str) or not spec['name'].strip():
errors.append('name required')
base = spec.get('base_url')
if not isinstance(base, str) or urllib.parse.urlsplit(base).scheme != 'https' or not host_of(base):
errors.append('base_url must be an https URL')
hosts = spec.get('allowed_hosts', [host_of(base)] if isinstance(base, str) else [])
if not isinstance(hosts, list) or not hosts or not all(isinstance(h, str) and h for h in hosts):
errors.append('allowed_hosts must be a non-empty list')
auth = spec.get('auth')
if not isinstance(auth, dict) or auth.get('type') not in AUTH_TYPES:
errors.append('auth.type must be one of ' + ', '.join(AUTH_TYPES))
else:
t = auth['type']
if t == 'api_key' and (auth.get('in') not in ('header', 'query') or not isinstance(auth.get('name'), str)):
errors.append('api_key auth needs in (header|query) and name')
if t == 'oauth2':
for k in ('authorize_url', 'token_url'):
if not isinstance(auth.get(k), str) or urllib.parse.urlsplit(auth[k]).scheme != 'https':
errors.append('oauth2 auth needs https ' + k)
if not isinstance(auth.get('scopes', []), list):
errors.append('oauth2 scopes must be a list')
creds = spec.get('credentials', [])
if not isinstance(creds, list) or not all(isinstance(c, dict) and isinstance(c.get('name'), str) and NAME_RE.fullmatch(c['name']) for c in creds):
errors.append('credentials must be a list of {name,label}')
cmds = spec.get('commands')
if not isinstance(cmds, dict) or not cmds:
errors.append('commands required')
cmds = {}
for name, c in cmds.items():
where = 'command ' + str(name)
if not isinstance(name, str) or not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,60}', name) or name in ('test', 'help', 'auth', 'list'):
errors.append(where + ': name must be lowercase (test/help/auth/list are reserved)')
if not isinstance(c, dict):
errors.append(where + ': must be an object')
continue
if c.get('method', 'GET') not in METHODS:
errors.append(where + ': method')
path = c.get('path')
if not isinstance(path, str) or not path.startswith('/') or '://' in path:
errors.append(where + ': path must start with / (relative to base_url)')
path = ''
params = c.get('params', {})
if not isinstance(params, dict):
errors.append(where + ': params must be an object')
params = {}
for p, d in params.items():
if not isinstance(d, dict) or d.get('in', 'query') not in PARAM_IN or d.get('type', 'string') not in PARAM_TYPES:
errors.append('%s param %s: in (%s), type (%s)' % (where, p, '|'.join(PARAM_IN), '|'.join(PARAM_TYPES)))
for p in re.findall(r'\{([A-Za-z0-9_]+)\}', path):
if params.get(p, {}).get('in') != 'path':
errors.append('%s: {%s} in path needs a path param' % (where, p))
out = c.get('output', {'type': 'json'})
if not isinstance(out, dict) or out.get('type', 'json') not in ('json', 'text', 'file'):
errors.append(where + ': output.type json|text|file')
if c.get('body_type', 'json') not in ('json', 'form'):
errors.append(where + ': body_type json|form')
test = spec.get('test')
if not isinstance(test, dict) or test.get('command') not in cmds:
errors.append('test.command must name a command')
elif is_write(cmds[test['command']]):
errors.append('test.command must be read-only')
return errors
def is_write(cmd):
if 'write' in cmd:
return bool(cmd['write'])
return cmd.get('method', 'GET') not in ('GET', 'HEAD')
def _convert(value, kind):
if kind == 'integer':
return int(value)
if kind == 'number':
return float(value)
if kind == 'boolean':
return value if isinstance(value, bool) else str(value).lower() in ('1', 'true', 'yes', 'on')
if kind in ('json', 'array'):
return json.loads(value) if isinstance(value, str) else value
return value if isinstance(value, str) else (json.dumps(value) if isinstance(value, (dict, list)) else str(value))
def build_request(spec, command, params, cred, allow_write=False):
"""(method, url, headers, body_bytes, output) for one call. Raises CallError with a plain sentence."""
cmds = spec.get('commands', {})
if command not in cmds:
raise CallError('Unknown command %r. Commands: %s' % (command, ', '.join(sorted(cmds))))
c = cmds[command]
if is_write(c) and not allow_write:
raise CallError('"%s" changes things at %s, and writing is switched off for this connection. Ask the person to '
'allow writing in Mesh → Settings → Connections.' % (command, spec.get('name')))
params = dict(params or {})
declared = c.get('params', {})
unknown = sorted(set(params) - set(declared))
if unknown:
raise CallError('Unknown parameter(s) %s for %s. Known: %s' % (', '.join(unknown), command, ', '.join(sorted(declared)) or 'none'))
path, query, body, headers = c['path'], [], {}, {'User-Agent': 'osmike-int/1', 'Accept': 'application/json'}
headers.update(spec.get('headers') or {})
headers.update(c.get('headers') or {})
for name, d in declared.items():
if name not in params:
if 'default' in d:
params[name] = d['default']
elif d.get('required'):
raise CallError('%s needs --%s (%s)' % (command, name, d.get('description') or d.get('type', 'string')))
else:
continue
try:
value = _convert(params[name], d.get('type', 'string'))
except (ValueError, TypeError):
raise CallError('--%s must be %s' % (name, d.get('type'))) from None
where, key = d.get('in', 'query'), d.get('name', name)
if where == 'path':
path = path.replace('{' + name + '}', urllib.parse.quote(str(value), safe=''))
elif where == 'query':
for v in (value if isinstance(value, list) else [value]):
query.append((key, str(v).lower() if isinstance(v, bool) else str(v)))
elif where == 'header':
headers[key] = str(value)
else:
body[key] = value
if c.get('body') and isinstance(c['body'], dict): # fixed body fields
body = {**c['body'], **body}
auth = spec.get('auth', {})
t = auth.get('type')
cred = cred or {}
def need(field):
v = cred.get(field)
if not v:
raise CallError('The connection has no %s. Ask the person to connect %s again.' % (field, spec.get('name')))
return v
if t == 'api_key':
value = auth.get('prefix', '') + need(auth.get('field', 'api_key'))
if auth['in'] == 'header':
headers[auth['name']] = value
else:
query.append((auth['name'], value))
elif t == 'bearer':
headers['Authorization'] = 'Bearer ' + need(auth.get('field', 'token'))
elif t == 'basic':
user = need(auth.get('username_field', 'username'))
pw = cred.get(auth.get('password_field', 'password'), '')
headers['Authorization'] = 'Basic ' + base64.b64encode(('%s:%s' % (user, pw)).encode()).decode()
elif t == 'oauth2':
headers['Authorization'] = (cred.get('token_type') or 'Bearer').capitalize() + ' ' + need('access_token')
url = spec['base_url'].rstrip('/') + path
if query:
url += ('&' if '?' in url else '?') + urllib.parse.urlencode(query)
allowed = [h.lower() for h in spec.get('allowed_hosts') or [host_of(spec['base_url'])]]
if host_of(url) not in allowed:
raise CallError('Host %s is not allowed for %s' % (host_of(url), spec.get('name')))
data = None
method = c.get('method', 'GET')
if body and method not in ('GET', 'HEAD'):
if c.get('body_type', 'json') == 'form':
data = urllib.parse.urlencode({k: (json.dumps(v) if isinstance(v, (dict, list)) else v) for k, v in body.items()}).encode()
headers['Content-Type'] = 'application/x-www-form-urlencoded'
else:
data = json.dumps(body).encode()
headers['Content-Type'] = 'application/json'
elif body:
raise CallError('%s is a %s request and cannot take body parameters' % (command, method))
return method, url, headers, data, c.get('output', {'type': 'json'})
class _NoRedirectOffHost(urllib.request.HTTPRedirectHandler):
def __init__(self, allowed):
self.allowed = allowed
def redirect_request(self, req, fp, code, msg, headers, newurl):
if host_of(newurl) not in self.allowed:
raise CallError('The service redirected to another host (%s); refused' % host_of(newurl))
return super().redirect_request(req, fp, code, msg, headers, newurl)
def call(spec, command, params, cred, allow_write=False, timeout=60, opener=None):
"""Run one command. Returns {status, ok, content_type, json|text|bytes}. Never raises for HTTP errors."""
method, url, headers, data, out = build_request(spec, command, params, cred, allow_write)
allowed = [h.lower() for h in spec.get('allowed_hosts') or [host_of(spec['base_url'])]]
op = opener or urllib.request.build_opener(_NoRedirectOffHost(allowed))
req = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
resp = op.open(req, timeout=timeout)
status, ctype, raw = resp.status, resp.headers.get('Content-Type', ''), resp.read(MAX_BYTES + 1)
except urllib.error.HTTPError as e:
status, ctype, raw = e.code, e.headers.get('Content-Type', '') if e.headers else '', e.read(200_000)
except urllib.error.URLError as e:
raise CallError('Could not reach %s: %s' % (host_of(url), e.reason)) from None
if len(raw) > MAX_BYTES:
raise CallError('The answer is larger than 25 MB')
result = {'status': status, 'ok': 200 <= status < 300, 'content_type': ctype.split(';')[0].strip(), 'output': out.get('type', 'json')}
if out.get('type') == 'file' and result['ok'] and 'json' not in ctype:
result['bytes'], result['ext'] = raw, out.get('ext') or (ctype.split('/')[-1].split(';')[0] or 'bin')
return result
text = raw.decode('utf-8', 'replace')
try:
result['json'] = json.loads(text) if text.strip() else None
except ValueError:
result['text'] = text[:200_000]
return result
def secret_values(cred):
return [v for v in (cred or {}).values() if isinstance(v, str) and len(v) >= 6]
def scrub(text, cred):
"""Remove every credential value from text (answers, errors) before an agent or model sees it."""
for v in sorted(secret_values(cred), key=len, reverse=True):
text = text.replace(v, '[secret]')
return text
def describe(spec):
"""Short help for agents: commands, parameters, which ones write."""
lines = ['%s (osmike-int %s <command> --param value)' % (spec.get('name'), spec.get('slug'))]
for name, c in sorted(spec.get('commands', {}).items()):
ps = []
for p, d in c.get('params', {}).items():
ps.append(('--%s' % p) + ('' if d.get('required') else '?'))
lines.append(' %-22s %s%s %s' % (name, ' '.join(ps), ' [writes]' if is_write(c) else '', c.get('summary', '')))
return '\n'.join(lines)
# ---------------------------------------------------------------- OAuth 2 helpers (stdlib)
def authorize_url(spec, cred, redirect_uri, state, challenge=None):
a = spec['auth']
q = {'response_type': 'code', 'client_id': cred.get('client_id', ''), 'redirect_uri': redirect_uri, 'state': state}
if a.get('scopes'):
q['scope'] = a.get('scope_separator', ' ').join(a['scopes'])
if a.get('pkce', True) and challenge:
q.update(code_challenge=challenge, code_challenge_method='S256')
q.update(a.get('extra_authorize_params') or {})
return a['authorize_url'] + ('&' if '?' in a['authorize_url'] else '?') + urllib.parse.urlencode(q)
def token_request(spec, cred, form, opener=None, timeout=30):
"""POST to the token URL (code exchange or refresh). Returns the JSON token answer or raises CallError."""
a = spec['auth']
headers = {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json', 'User-Agent': 'osmike-int/1'}
form = dict(form)
if a.get('client_auth', 'body') == 'basic':
headers['Authorization'] = 'Basic ' + base64.b64encode(('%s:%s' % (cred.get('client_id', ''), cred.get('client_secret', ''))).encode()).decode()
else:
form['client_id'] = cred.get('client_id', '')
if cred.get('client_secret'):
form['client_secret'] = cred['client_secret']
req = urllib.request.Request(a['token_url'], data=urllib.parse.urlencode(form).encode(), headers=headers, method='POST')
try:
raw = (opener or urllib.request.build_opener()).open(req, timeout=timeout).read(1_000_000)
except urllib.error.HTTPError as e:
raise CallError('The token request was refused (%s): %s' % (e.code, scrub(e.read(2000).decode('utf-8', 'replace'), cred)[:300])) from None
except urllib.error.URLError as e:
raise CallError('Could not reach the token URL: %s' % e.reason) from None
text = raw.decode('utf-8', 'replace')
try:
data = json.loads(text)
except ValueError:
data = dict(urllib.parse.parse_qsl(text)) # some providers answer form-encoded
if not data.get('access_token'):
raise CallError('The provider gave no access token: %s' % scrub(json.dumps(data), cred)[:300])
return data