331 lines
16 KiB
Python
331 lines
16 KiB
Python
"""Generic integration engine (Mike, 2026-11-13: "it will be thousands of integrations").
|
|
|
|
One integration = one declarative spec (JSON). This module turns (spec, command, params, credential) into an HTTP call.
|
|
The same file is used by osmike-runs (server-side tests, trials) and by the `osmike-int` CLI in every agent.
|
|
Stdlib only. Specs are data, never code.
|
|
|
|
Spec:
|
|
{
|
|
"slug": "elevenlabs", "name": "ElevenLabs", "version": 1,
|
|
"base_url": "https://api.elevenlabs.io",
|
|
"allowed_hosts": ["api.elevenlabs.io"], # optional; defaults to base_url's host
|
|
"auth": {"type": "api_key", "in": "header", "name": "xi-api-key", "field": "api_key"}
|
|
| {"type": "bearer", "field": "token"}
|
|
| {"type": "basic", "username_field": "username", "password_field": "password"}
|
|
| {"type": "none"}
|
|
| {"type": "oauth2", "authorize_url": "...", "token_url": "...", "scopes": ["..."], "scope_separator": " ",
|
|
"pkce": true, "client_auth": "body" | "basic", "extra_authorize_params": {}},
|
|
"credentials": [{"name": "api_key", "label": "API key"}], # what the person gives (oauth2: client_id, client_secret)
|
|
"setup": "Where to get the key / how to create the OAuth app. {redirect_uri} is filled in.",
|
|
"headers": {"Accept": "application/json"}, # optional fixed headers
|
|
"test": {"command": "me", "params": {}}, # read-only call proving the credential works
|
|
"commands": {
|
|
"voices": {"summary": "List voices", "method": "GET", "path": "/v1/voices",
|
|
"params": {"page_size": {"in": "query", "type": "integer", "description": "..."}}},
|
|
"speak": {"summary": "Text to speech", "method": "POST", "path": "/v1/text-to-speech/{voice_id}", "write": false,
|
|
"params": {"voice_id": {"in": "path", "required": true}, "text": {"in": "body", "required": true}},
|
|
"body_type": "json", "output": {"type": "file", "ext": "mp3"}}
|
|
}
|
|
}
|
|
"write" defaults to true for every method except GET/HEAD; a write command needs the connection's "allow writing".
|
|
"""
|
|
import base64
|
|
import json
|
|
import re
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
AUTH_TYPES = ('api_key', 'bearer', 'basic', 'none', 'oauth2')
|
|
METHODS = ('GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE')
|
|
PARAM_IN = ('query', 'path', 'body', 'header')
|
|
PARAM_TYPES = ('string', 'integer', 'number', 'boolean', 'json', 'array')
|
|
SLUG_RE = re.compile(r'[a-z0-9]+(?:-[a-z0-9]+)*')
|
|
NAME_RE = re.compile(r'[A-Za-z0-9_.-]{1,80}')
|
|
MAX_BYTES = 25 * 1024 * 1024
|
|
|
|
|
|
class SpecError(ValueError):
|
|
pass
|
|
|
|
|
|
class CallError(ValueError):
|
|
pass
|
|
|
|
|
|
def host_of(url):
|
|
return (urllib.parse.urlsplit(url).hostname or '').lower()
|
|
|
|
|
|
def validate(spec):
|
|
"""List of problems (empty = valid). Strict on the shape so a bad spec never reaches an agent."""
|
|
errors = []
|
|
if not isinstance(spec, dict):
|
|
return ['spec must be an object']
|
|
slug = spec.get('slug')
|
|
if not isinstance(slug, str) or not SLUG_RE.fullmatch(slug) or len(slug) > 60:
|
|
errors.append('slug: lowercase letters, digits and dashes')
|
|
if not isinstance(spec.get('name'), str) or not spec['name'].strip():
|
|
errors.append('name required')
|
|
base = spec.get('base_url')
|
|
if not isinstance(base, str) or urllib.parse.urlsplit(base).scheme != 'https' or not host_of(base):
|
|
errors.append('base_url must be an https URL')
|
|
hosts = spec.get('allowed_hosts', [host_of(base)] if isinstance(base, str) else [])
|
|
if not isinstance(hosts, list) or not hosts or not all(isinstance(h, str) and h for h in hosts):
|
|
errors.append('allowed_hosts must be a non-empty list')
|
|
auth = spec.get('auth')
|
|
if not isinstance(auth, dict) or auth.get('type') not in AUTH_TYPES:
|
|
errors.append('auth.type must be one of ' + ', '.join(AUTH_TYPES))
|
|
else:
|
|
t = auth['type']
|
|
if t == 'api_key' and (auth.get('in') not in ('header', 'query') or not isinstance(auth.get('name'), str)):
|
|
errors.append('api_key auth needs in (header|query) and name')
|
|
if t == 'oauth2':
|
|
for k in ('authorize_url', 'token_url'):
|
|
if not isinstance(auth.get(k), str) or urllib.parse.urlsplit(auth[k]).scheme != 'https':
|
|
errors.append('oauth2 auth needs https ' + k)
|
|
if not isinstance(auth.get('scopes', []), list):
|
|
errors.append('oauth2 scopes must be a list')
|
|
creds = spec.get('credentials', [])
|
|
if not isinstance(creds, list) or not all(isinstance(c, dict) and isinstance(c.get('name'), str) and NAME_RE.fullmatch(c['name']) for c in creds):
|
|
errors.append('credentials must be a list of {name,label}')
|
|
cmds = spec.get('commands')
|
|
if not isinstance(cmds, dict) or not cmds:
|
|
errors.append('commands required')
|
|
cmds = {}
|
|
for name, c in cmds.items():
|
|
where = 'command ' + str(name)
|
|
if not isinstance(name, str) or not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,60}', name) or name in ('test', 'help', 'auth', 'list'):
|
|
errors.append(where + ': name must be lowercase (test/help/auth/list are reserved)')
|
|
if not isinstance(c, dict):
|
|
errors.append(where + ': must be an object')
|
|
continue
|
|
if c.get('method', 'GET') not in METHODS:
|
|
errors.append(where + ': method')
|
|
path = c.get('path')
|
|
if not isinstance(path, str) or not path.startswith('/') or '://' in path:
|
|
errors.append(where + ': path must start with / (relative to base_url)')
|
|
path = ''
|
|
params = c.get('params', {})
|
|
if not isinstance(params, dict):
|
|
errors.append(where + ': params must be an object')
|
|
params = {}
|
|
for p, d in params.items():
|
|
if not isinstance(d, dict) or d.get('in', 'query') not in PARAM_IN or d.get('type', 'string') not in PARAM_TYPES:
|
|
errors.append('%s param %s: in (%s), type (%s)' % (where, p, '|'.join(PARAM_IN), '|'.join(PARAM_TYPES)))
|
|
for p in re.findall(r'\{([A-Za-z0-9_]+)\}', path):
|
|
if params.get(p, {}).get('in') != 'path':
|
|
errors.append('%s: {%s} in path needs a path param' % (where, p))
|
|
out = c.get('output', {'type': 'json'})
|
|
if not isinstance(out, dict) or out.get('type', 'json') not in ('json', 'text', 'file'):
|
|
errors.append(where + ': output.type json|text|file')
|
|
if c.get('body_type', 'json') not in ('json', 'form'):
|
|
errors.append(where + ': body_type json|form')
|
|
test = spec.get('test')
|
|
if not isinstance(test, dict) or test.get('command') not in cmds:
|
|
errors.append('test.command must name a command')
|
|
elif is_write(cmds[test['command']]):
|
|
errors.append('test.command must be read-only')
|
|
return errors
|
|
|
|
|
|
def is_write(cmd):
|
|
if 'write' in cmd:
|
|
return bool(cmd['write'])
|
|
return cmd.get('method', 'GET') not in ('GET', 'HEAD')
|
|
|
|
|
|
def _convert(value, kind):
|
|
if kind == 'integer':
|
|
return int(value)
|
|
if kind == 'number':
|
|
return float(value)
|
|
if kind == 'boolean':
|
|
return value if isinstance(value, bool) else str(value).lower() in ('1', 'true', 'yes', 'on')
|
|
if kind in ('json', 'array'):
|
|
return json.loads(value) if isinstance(value, str) else value
|
|
return value if isinstance(value, str) else (json.dumps(value) if isinstance(value, (dict, list)) else str(value))
|
|
|
|
|
|
def build_request(spec, command, params, cred, allow_write=False):
|
|
"""(method, url, headers, body_bytes, output) for one call. Raises CallError with a plain sentence."""
|
|
cmds = spec.get('commands', {})
|
|
if command not in cmds:
|
|
raise CallError('Unknown command %r. Commands: %s' % (command, ', '.join(sorted(cmds))))
|
|
c = cmds[command]
|
|
if is_write(c) and not allow_write:
|
|
raise CallError('"%s" changes things at %s, and writing is switched off for this connection. Ask the person to '
|
|
'allow writing in Mesh → Settings → Connections.' % (command, spec.get('name')))
|
|
params = dict(params or {})
|
|
declared = c.get('params', {})
|
|
unknown = sorted(set(params) - set(declared))
|
|
if unknown:
|
|
raise CallError('Unknown parameter(s) %s for %s. Known: %s' % (', '.join(unknown), command, ', '.join(sorted(declared)) or 'none'))
|
|
path, query, body, headers = c['path'], [], {}, {'User-Agent': 'osmike-int/1', 'Accept': 'application/json'}
|
|
headers.update(spec.get('headers') or {})
|
|
headers.update(c.get('headers') or {})
|
|
for name, d in declared.items():
|
|
if name not in params:
|
|
if 'default' in d:
|
|
params[name] = d['default']
|
|
elif d.get('required'):
|
|
raise CallError('%s needs --%s (%s)' % (command, name, d.get('description') or d.get('type', 'string')))
|
|
else:
|
|
continue
|
|
try:
|
|
value = _convert(params[name], d.get('type', 'string'))
|
|
except (ValueError, TypeError):
|
|
raise CallError('--%s must be %s' % (name, d.get('type'))) from None
|
|
where, key = d.get('in', 'query'), d.get('name', name)
|
|
if where == 'path':
|
|
path = path.replace('{' + name + '}', urllib.parse.quote(str(value), safe=''))
|
|
elif where == 'query':
|
|
for v in (value if isinstance(value, list) else [value]):
|
|
query.append((key, str(v).lower() if isinstance(v, bool) else str(v)))
|
|
elif where == 'header':
|
|
headers[key] = str(value)
|
|
else:
|
|
body[key] = value
|
|
if c.get('body') and isinstance(c['body'], dict): # fixed body fields
|
|
body = {**c['body'], **body}
|
|
auth = spec.get('auth', {})
|
|
t = auth.get('type')
|
|
cred = cred or {}
|
|
|
|
def need(field):
|
|
v = cred.get(field)
|
|
if not v:
|
|
raise CallError('The connection has no %s. Ask the person to connect %s again.' % (field, spec.get('name')))
|
|
return v
|
|
if t == 'api_key':
|
|
value = auth.get('prefix', '') + need(auth.get('field', 'api_key'))
|
|
if auth['in'] == 'header':
|
|
headers[auth['name']] = value
|
|
else:
|
|
query.append((auth['name'], value))
|
|
elif t == 'bearer':
|
|
headers['Authorization'] = 'Bearer ' + need(auth.get('field', 'token'))
|
|
elif t == 'basic':
|
|
user = need(auth.get('username_field', 'username'))
|
|
pw = cred.get(auth.get('password_field', 'password'), '')
|
|
headers['Authorization'] = 'Basic ' + base64.b64encode(('%s:%s' % (user, pw)).encode()).decode()
|
|
elif t == 'oauth2':
|
|
headers['Authorization'] = (cred.get('token_type') or 'Bearer').capitalize() + ' ' + need('access_token')
|
|
url = spec['base_url'].rstrip('/') + path
|
|
if query:
|
|
url += ('&' if '?' in url else '?') + urllib.parse.urlencode(query)
|
|
allowed = [h.lower() for h in spec.get('allowed_hosts') or [host_of(spec['base_url'])]]
|
|
if host_of(url) not in allowed:
|
|
raise CallError('Host %s is not allowed for %s' % (host_of(url), spec.get('name')))
|
|
data = None
|
|
method = c.get('method', 'GET')
|
|
if body and method not in ('GET', 'HEAD'):
|
|
if c.get('body_type', 'json') == 'form':
|
|
data = urllib.parse.urlencode({k: (json.dumps(v) if isinstance(v, (dict, list)) else v) for k, v in body.items()}).encode()
|
|
headers['Content-Type'] = 'application/x-www-form-urlencoded'
|
|
else:
|
|
data = json.dumps(body).encode()
|
|
headers['Content-Type'] = 'application/json'
|
|
elif body:
|
|
raise CallError('%s is a %s request and cannot take body parameters' % (command, method))
|
|
return method, url, headers, data, c.get('output', {'type': 'json'})
|
|
|
|
|
|
class _NoRedirectOffHost(urllib.request.HTTPRedirectHandler):
|
|
def __init__(self, allowed):
|
|
self.allowed = allowed
|
|
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
if host_of(newurl) not in self.allowed:
|
|
raise CallError('The service redirected to another host (%s); refused' % host_of(newurl))
|
|
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
|
|
|
|
def call(spec, command, params, cred, allow_write=False, timeout=60, opener=None):
|
|
"""Run one command. Returns {status, ok, content_type, json|text|bytes}. Never raises for HTTP errors."""
|
|
method, url, headers, data, out = build_request(spec, command, params, cred, allow_write)
|
|
allowed = [h.lower() for h in spec.get('allowed_hosts') or [host_of(spec['base_url'])]]
|
|
op = opener or urllib.request.build_opener(_NoRedirectOffHost(allowed))
|
|
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
|
try:
|
|
resp = op.open(req, timeout=timeout)
|
|
status, ctype, raw = resp.status, resp.headers.get('Content-Type', ''), resp.read(MAX_BYTES + 1)
|
|
except urllib.error.HTTPError as e:
|
|
status, ctype, raw = e.code, e.headers.get('Content-Type', '') if e.headers else '', e.read(200_000)
|
|
except urllib.error.URLError as e:
|
|
raise CallError('Could not reach %s: %s' % (host_of(url), e.reason)) from None
|
|
if len(raw) > MAX_BYTES:
|
|
raise CallError('The answer is larger than 25 MB')
|
|
result = {'status': status, 'ok': 200 <= status < 300, 'content_type': ctype.split(';')[0].strip(), 'output': out.get('type', 'json')}
|
|
if out.get('type') == 'file' and result['ok'] and 'json' not in ctype:
|
|
result['bytes'], result['ext'] = raw, out.get('ext') or (ctype.split('/')[-1].split(';')[0] or 'bin')
|
|
return result
|
|
text = raw.decode('utf-8', 'replace')
|
|
try:
|
|
result['json'] = json.loads(text) if text.strip() else None
|
|
except ValueError:
|
|
result['text'] = text[:200_000]
|
|
return result
|
|
|
|
|
|
def secret_values(cred):
|
|
return [v for v in (cred or {}).values() if isinstance(v, str) and len(v) >= 6]
|
|
|
|
|
|
def scrub(text, cred):
|
|
"""Remove every credential value from text (answers, errors) before an agent or model sees it."""
|
|
for v in sorted(secret_values(cred), key=len, reverse=True):
|
|
text = text.replace(v, '[secret]')
|
|
return text
|
|
|
|
|
|
def describe(spec):
|
|
"""Short help for agents: commands, parameters, which ones write."""
|
|
lines = ['%s (osmike-int %s <command> --param value)' % (spec.get('name'), spec.get('slug'))]
|
|
for name, c in sorted(spec.get('commands', {}).items()):
|
|
ps = []
|
|
for p, d in c.get('params', {}).items():
|
|
ps.append(('--%s' % p) + ('' if d.get('required') else '?'))
|
|
lines.append(' %-22s %s%s %s' % (name, ' '.join(ps), ' [writes]' if is_write(c) else '', c.get('summary', '')))
|
|
return '\n'.join(lines)
|
|
|
|
|
|
# ---------------------------------------------------------------- OAuth 2 helpers (stdlib)
|
|
def authorize_url(spec, cred, redirect_uri, state, challenge=None):
|
|
a = spec['auth']
|
|
q = {'response_type': 'code', 'client_id': cred.get('client_id', ''), 'redirect_uri': redirect_uri, 'state': state}
|
|
if a.get('scopes'):
|
|
q['scope'] = a.get('scope_separator', ' ').join(a['scopes'])
|
|
if a.get('pkce', True) and challenge:
|
|
q.update(code_challenge=challenge, code_challenge_method='S256')
|
|
q.update(a.get('extra_authorize_params') or {})
|
|
return a['authorize_url'] + ('&' if '?' in a['authorize_url'] else '?') + urllib.parse.urlencode(q)
|
|
|
|
|
|
def token_request(spec, cred, form, opener=None, timeout=30):
|
|
"""POST to the token URL (code exchange or refresh). Returns the JSON token answer or raises CallError."""
|
|
a = spec['auth']
|
|
headers = {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json', 'User-Agent': 'osmike-int/1'}
|
|
form = dict(form)
|
|
if a.get('client_auth', 'body') == 'basic':
|
|
headers['Authorization'] = 'Basic ' + base64.b64encode(('%s:%s' % (cred.get('client_id', ''), cred.get('client_secret', ''))).encode()).decode()
|
|
else:
|
|
form['client_id'] = cred.get('client_id', '')
|
|
if cred.get('client_secret'):
|
|
form['client_secret'] = cred['client_secret']
|
|
req = urllib.request.Request(a['token_url'], data=urllib.parse.urlencode(form).encode(), headers=headers, method='POST')
|
|
try:
|
|
raw = (opener or urllib.request.build_opener()).open(req, timeout=timeout).read(1_000_000)
|
|
except urllib.error.HTTPError as e:
|
|
raise CallError('The token request was refused (%s): %s' % (e.code, scrub(e.read(2000).decode('utf-8', 'replace'), cred)[:300])) from None
|
|
except urllib.error.URLError as e:
|
|
raise CallError('Could not reach the token URL: %s' % e.reason) from None
|
|
text = raw.decode('utf-8', 'replace')
|
|
try:
|
|
data = json.loads(text)
|
|
except ValueError:
|
|
data = dict(urllib.parse.parse_qsl(text)) # some providers answer form-encoded
|
|
if not data.get('access_token'):
|
|
raise CallError('The provider gave no access token: %s' % scrub(json.dumps(data), cred)[:300])
|
|
return data
|