"""Generic integration engine (Mike, 2026-11-13: "it will be thousands of integrations"). One integration = one declarative spec (JSON). This module turns (spec, command, params, credential) into an HTTP call. The same file is used by osmike-runs (server-side tests, trials) and by the `osmike-int` CLI in every agent. Stdlib only. Specs are data, never code. Spec: { "slug": "elevenlabs", "name": "ElevenLabs", "version": 1, "base_url": "https://api.elevenlabs.io", "allowed_hosts": ["api.elevenlabs.io"], # optional; defaults to base_url's host "auth": {"type": "api_key", "in": "header", "name": "xi-api-key", "field": "api_key"} | {"type": "bearer", "field": "token"} | {"type": "basic", "username_field": "username", "password_field": "password"} | {"type": "none"} | {"type": "oauth2", "authorize_url": "...", "token_url": "...", "scopes": ["..."], "scope_separator": " ", "pkce": true, "client_auth": "body" | "basic", "extra_authorize_params": {}}, "credentials": [{"name": "api_key", "label": "API key"}], # what the person gives (oauth2: client_id, client_secret) "setup": "Where to get the key / how to create the OAuth app. {redirect_uri} is filled in.", "headers": {"Accept": "application/json"}, # optional fixed headers "test": {"command": "me", "params": {}}, # read-only call proving the credential works "commands": { "voices": {"summary": "List voices", "method": "GET", "path": "/v1/voices", "params": {"page_size": {"in": "query", "type": "integer", "description": "..."}}}, "speak": {"summary": "Text to speech", "method": "POST", "path": "/v1/text-to-speech/{voice_id}", "write": false, "params": {"voice_id": {"in": "path", "required": true}, "text": {"in": "body", "required": true}}, "body_type": "json", "output": {"type": "file", "ext": "mp3"}} } } "write" defaults to true for every method except GET/HEAD; a write command needs the connection's "allow writing". """ import base64 import json import re import urllib.error import urllib.parse import urllib.request AUTH_TYPES = ('api_key', 'bearer', 'basic', 'none', 'oauth2') METHODS = ('GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE') PARAM_IN = ('query', 'path', 'body', 'header') PARAM_TYPES = ('string', 'integer', 'number', 'boolean', 'json', 'array') SLUG_RE = re.compile(r'[a-z0-9]+(?:-[a-z0-9]+)*') NAME_RE = re.compile(r'[A-Za-z0-9_.-]{1,80}') MAX_BYTES = 25 * 1024 * 1024 class SpecError(ValueError): pass class CallError(ValueError): pass def host_of(url): return (urllib.parse.urlsplit(url).hostname or '').lower() def validate(spec): """List of problems (empty = valid). Strict on the shape so a bad spec never reaches an agent.""" errors = [] if not isinstance(spec, dict): return ['spec must be an object'] slug = spec.get('slug') if not isinstance(slug, str) or not SLUG_RE.fullmatch(slug) or len(slug) > 60: errors.append('slug: lowercase letters, digits and dashes') if not isinstance(spec.get('name'), str) or not spec['name'].strip(): errors.append('name required') base = spec.get('base_url') if not isinstance(base, str) or urllib.parse.urlsplit(base).scheme != 'https' or not host_of(base): errors.append('base_url must be an https URL') hosts = spec.get('allowed_hosts', [host_of(base)] if isinstance(base, str) else []) if not isinstance(hosts, list) or not hosts or not all(isinstance(h, str) and h for h in hosts): errors.append('allowed_hosts must be a non-empty list') auth = spec.get('auth') if not isinstance(auth, dict) or auth.get('type') not in AUTH_TYPES: errors.append('auth.type must be one of ' + ', '.join(AUTH_TYPES)) else: t = auth['type'] if t == 'api_key' and (auth.get('in') not in ('header', 'query') or not isinstance(auth.get('name'), str)): errors.append('api_key auth needs in (header|query) and name') if t == 'oauth2': for k in ('authorize_url', 'token_url'): if not isinstance(auth.get(k), str) or urllib.parse.urlsplit(auth[k]).scheme != 'https': errors.append('oauth2 auth needs https ' + k) if not isinstance(auth.get('scopes', []), list): errors.append('oauth2 scopes must be a list') creds = spec.get('credentials', []) if not isinstance(creds, list) or not all(isinstance(c, dict) and isinstance(c.get('name'), str) and NAME_RE.fullmatch(c['name']) for c in creds): errors.append('credentials must be a list of {name,label}') cmds = spec.get('commands') if not isinstance(cmds, dict) or not cmds: errors.append('commands required') cmds = {} for name, c in cmds.items(): where = 'command ' + str(name) if not isinstance(name, str) or not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,60}', name) or name in ('test', 'help', 'auth', 'list'): errors.append(where + ': name must be lowercase (test/help/auth/list are reserved)') if not isinstance(c, dict): errors.append(where + ': must be an object') continue if c.get('method', 'GET') not in METHODS: errors.append(where + ': method') path = c.get('path') if not isinstance(path, str) or not path.startswith('/') or '://' in path: errors.append(where + ': path must start with / (relative to base_url)') path = '' params = c.get('params', {}) if not isinstance(params, dict): errors.append(where + ': params must be an object') params = {} for p, d in params.items(): if not isinstance(d, dict) or d.get('in', 'query') not in PARAM_IN or d.get('type', 'string') not in PARAM_TYPES: errors.append('%s param %s: in (%s), type (%s)' % (where, p, '|'.join(PARAM_IN), '|'.join(PARAM_TYPES))) for p in re.findall(r'\{([A-Za-z0-9_]+)\}', path): if params.get(p, {}).get('in') != 'path': errors.append('%s: {%s} in path needs a path param' % (where, p)) out = c.get('output', {'type': 'json'}) if not isinstance(out, dict) or out.get('type', 'json') not in ('json', 'text', 'file'): errors.append(where + ': output.type json|text|file') if c.get('body_type', 'json') not in ('json', 'form'): errors.append(where + ': body_type json|form') test = spec.get('test') if not isinstance(test, dict) or test.get('command') not in cmds: errors.append('test.command must name a command') elif is_write(cmds[test['command']]): errors.append('test.command must be read-only') return errors def is_write(cmd): if 'write' in cmd: return bool(cmd['write']) return cmd.get('method', 'GET') not in ('GET', 'HEAD') def _convert(value, kind): if kind == 'integer': return int(value) if kind == 'number': return float(value) if kind == 'boolean': return value if isinstance(value, bool) else str(value).lower() in ('1', 'true', 'yes', 'on') if kind in ('json', 'array'): return json.loads(value) if isinstance(value, str) else value return value if isinstance(value, str) else (json.dumps(value) if isinstance(value, (dict, list)) else str(value)) def build_request(spec, command, params, cred, allow_write=False): """(method, url, headers, body_bytes, output) for one call. Raises CallError with a plain sentence.""" cmds = spec.get('commands', {}) if command not in cmds: raise CallError('Unknown command %r. Commands: %s' % (command, ', '.join(sorted(cmds)))) c = cmds[command] if is_write(c) and not allow_write: raise CallError('"%s" changes things at %s, and writing is switched off for this connection. Ask the person to ' 'allow writing in Mesh → Settings → OsMike Connect.' % (command, spec.get('name'))) params = dict(params or {}) declared = c.get('params', {}) unknown = sorted(set(params) - set(declared)) if unknown: raise CallError('Unknown parameter(s) %s for %s. Known: %s' % (', '.join(unknown), command, ', '.join(sorted(declared)) or 'none')) path, query, body, headers = c['path'], [], {}, {'User-Agent': 'osmike-int/1', 'Accept': 'application/json'} headers.update(spec.get('headers') or {}) headers.update(c.get('headers') or {}) for name, d in declared.items(): if name not in params: if 'default' in d: params[name] = d['default'] elif d.get('required'): raise CallError('%s needs --%s (%s)' % (command, name, d.get('description') or d.get('type', 'string'))) else: continue try: value = _convert(params[name], d.get('type', 'string')) except (ValueError, TypeError): raise CallError('--%s must be %s' % (name, d.get('type'))) from None where, key = d.get('in', 'query'), d.get('name', name) if where == 'path': path = path.replace('{' + name + '}', urllib.parse.quote(str(value), safe='')) elif where == 'query': for v in (value if isinstance(value, list) else [value]): query.append((key, str(v).lower() if isinstance(v, bool) else str(v))) elif where == 'header': headers[key] = str(value) else: body[key] = value if c.get('body') and isinstance(c['body'], dict): # fixed body fields body = {**c['body'], **body} auth = spec.get('auth', {}) t = auth.get('type') cred = cred or {} def need(field): v = cred.get(field) if not v: raise CallError('The connection has no %s. Ask the person to connect %s again.' % (field, spec.get('name'))) return v if t == 'api_key': value = auth.get('prefix', '') + need(auth.get('field', 'api_key')) if auth['in'] == 'header': headers[auth['name']] = value else: query.append((auth['name'], value)) elif t == 'bearer': headers['Authorization'] = 'Bearer ' + need(auth.get('field', 'token')) elif t == 'basic': user = need(auth.get('username_field', 'username')) pw = cred.get(auth.get('password_field', 'password'), '') headers['Authorization'] = 'Basic ' + base64.b64encode(('%s:%s' % (user, pw)).encode()).decode() elif t == 'oauth2': headers['Authorization'] = (cred.get('token_type') or 'Bearer').capitalize() + ' ' + need('access_token') url = spec['base_url'].rstrip('/') + path if query: url += ('&' if '?' in url else '?') + urllib.parse.urlencode(query) allowed = [h.lower() for h in spec.get('allowed_hosts') or [host_of(spec['base_url'])]] if host_of(url) not in allowed: raise CallError('Host %s is not allowed for %s' % (host_of(url), spec.get('name'))) data = None method = c.get('method', 'GET') if body and method not in ('GET', 'HEAD'): if c.get('body_type', 'json') == 'form': data = urllib.parse.urlencode({k: (json.dumps(v) if isinstance(v, (dict, list)) else v) for k, v in body.items()}).encode() headers['Content-Type'] = 'application/x-www-form-urlencoded' else: data = json.dumps(body).encode() headers['Content-Type'] = 'application/json' elif body: raise CallError('%s is a %s request and cannot take body parameters' % (command, method)) return method, url, headers, data, c.get('output', {'type': 'json'}) class _NoRedirectOffHost(urllib.request.HTTPRedirectHandler): def __init__(self, allowed): self.allowed = allowed def redirect_request(self, req, fp, code, msg, headers, newurl): if host_of(newurl) not in self.allowed: raise CallError('The service redirected to another host (%s); refused' % host_of(newurl)) return super().redirect_request(req, fp, code, msg, headers, newurl) def call(spec, command, params, cred, allow_write=False, timeout=60, opener=None): """Run one command. Returns {status, ok, content_type, json|text|bytes}. Never raises for HTTP errors.""" method, url, headers, data, out = build_request(spec, command, params, cred, allow_write) allowed = [h.lower() for h in spec.get('allowed_hosts') or [host_of(spec['base_url'])]] op = opener or urllib.request.build_opener(_NoRedirectOffHost(allowed)) req = urllib.request.Request(url, data=data, headers=headers, method=method) try: resp = op.open(req, timeout=timeout) status, ctype, raw = resp.status, resp.headers.get('Content-Type', ''), resp.read(MAX_BYTES + 1) except urllib.error.HTTPError as e: status, ctype, raw = e.code, e.headers.get('Content-Type', '') if e.headers else '', e.read(200_000) except urllib.error.URLError as e: raise CallError('Could not reach %s: %s' % (host_of(url), e.reason)) from None if len(raw) > MAX_BYTES: raise CallError('The answer is larger than 25 MB') result = {'status': status, 'ok': 200 <= status < 300, 'content_type': ctype.split(';')[0].strip(), 'output': out.get('type', 'json')} if out.get('type') == 'file' and result['ok'] and 'json' not in ctype: result['bytes'], result['ext'] = raw, out.get('ext') or (ctype.split('/')[-1].split(';')[0] or 'bin') return result text = raw.decode('utf-8', 'replace') try: result['json'] = json.loads(text) if text.strip() else None except ValueError: result['text'] = text[:200_000] return result def secret_values(cred): return [v for v in (cred or {}).values() if isinstance(v, str) and len(v) >= 6] def scrub(text, cred): """Remove every credential value from text (answers, errors) before an agent or model sees it.""" for v in sorted(secret_values(cred), key=len, reverse=True): text = text.replace(v, '[secret]') return text def describe(spec): """Short help for agents: commands, parameters, which ones write.""" lines = ['%s (osmike-int %s --param value)' % (spec.get('name'), spec.get('slug'))] for name, c in sorted(spec.get('commands', {}).items()): ps = [] for p, d in c.get('params', {}).items(): ps.append(('--%s' % p) + ('' if d.get('required') else '?')) lines.append(' %-22s %s%s %s' % (name, ' '.join(ps), ' [writes]' if is_write(c) else '', c.get('summary', ''))) return '\n'.join(lines) # ---------------------------------------------------------------- OAuth 2 helpers (stdlib) def authorize_url(spec, cred, redirect_uri, state, challenge=None): a = spec['auth'] q = {'response_type': 'code', 'client_id': cred.get('client_id', ''), 'redirect_uri': redirect_uri, 'state': state} if a.get('scopes'): q['scope'] = a.get('scope_separator', ' ').join(a['scopes']) if a.get('pkce', True) and challenge: q.update(code_challenge=challenge, code_challenge_method='S256') q.update(a.get('extra_authorize_params') or {}) return a['authorize_url'] + ('&' if '?' in a['authorize_url'] else '?') + urllib.parse.urlencode(q) def token_request(spec, cred, form, opener=None, timeout=30): """POST to the token URL (code exchange or refresh). Returns the JSON token answer or raises CallError.""" a = spec['auth'] headers = {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json', 'User-Agent': 'osmike-int/1'} form = dict(form) if a.get('client_auth', 'body') == 'basic': headers['Authorization'] = 'Basic ' + base64.b64encode(('%s:%s' % (cred.get('client_id', ''), cred.get('client_secret', ''))).encode()).decode() else: form['client_id'] = cred.get('client_id', '') if cred.get('client_secret'): form['client_secret'] = cred['client_secret'] req = urllib.request.Request(a['token_url'], data=urllib.parse.urlencode(form).encode(), headers=headers, method='POST') try: raw = (opener or urllib.request.build_opener()).open(req, timeout=timeout).read(1_000_000) except urllib.error.HTTPError as e: raise CallError('The token request was refused (%s): %s' % (e.code, scrub(e.read(2000).decode('utf-8', 'replace'), cred)[:300])) from None except urllib.error.URLError as e: raise CallError('Could not reach the token URL: %s' % e.reason) from None text = raw.decode('utf-8', 'replace') try: data = json.loads(text) except ValueError: data = dict(urllib.parse.parse_qsl(text)) # some providers answer form-encoded if not data.get('access_token'): raise CallError('The provider gave no access token: %s' % scrub(json.dumps(data), cred)[:300]) return data