osmike-int: use integrated services (calls run server side with the connection keys)

This commit is contained in:
OsMike-Agent 2026-11-13 17:46:30 +00:00
parent 126992649c
commit 405e44ae05
2 changed files with 534 additions and 0 deletions

330
intengine.py Normal file
View File

@ -0,0 +1,330 @@
"""Generic integration engine (Mike, 2026-11-13: "it will be thousands of integrations").
One integration = one declarative spec (JSON). This module turns (spec, command, params, credential) into an HTTP call.
The same file is used by osmike-runs (server-side tests, trials) and by the `osmike-int` CLI in every agent.
Stdlib only. Specs are data, never code.
Spec:
{
"slug": "elevenlabs", "name": "ElevenLabs", "version": 1,
"base_url": "https://api.elevenlabs.io",
"allowed_hosts": ["api.elevenlabs.io"], # optional; defaults to base_url's host
"auth": {"type": "api_key", "in": "header", "name": "xi-api-key", "field": "api_key"}
| {"type": "bearer", "field": "token"}
| {"type": "basic", "username_field": "username", "password_field": "password"}
| {"type": "none"}
| {"type": "oauth2", "authorize_url": "...", "token_url": "...", "scopes": ["..."], "scope_separator": " ",
"pkce": true, "client_auth": "body" | "basic", "extra_authorize_params": {}},
"credentials": [{"name": "api_key", "label": "API key"}], # what the person gives (oauth2: client_id, client_secret)
"setup": "Where to get the key / how to create the OAuth app. {redirect_uri} is filled in.",
"headers": {"Accept": "application/json"}, # optional fixed headers
"test": {"command": "me", "params": {}}, # read-only call proving the credential works
"commands": {
"voices": {"summary": "List voices", "method": "GET", "path": "/v1/voices",
"params": {"page_size": {"in": "query", "type": "integer", "description": "..."}}},
"speak": {"summary": "Text to speech", "method": "POST", "path": "/v1/text-to-speech/{voice_id}", "write": false,
"params": {"voice_id": {"in": "path", "required": true}, "text": {"in": "body", "required": true}},
"body_type": "json", "output": {"type": "file", "ext": "mp3"}}
}
}
"write" defaults to true for every method except GET/HEAD; a write command needs the connection's "allow writing".
"""
import base64
import json
import re
import urllib.error
import urllib.parse
import urllib.request
AUTH_TYPES = ('api_key', 'bearer', 'basic', 'none', 'oauth2')
METHODS = ('GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE')
PARAM_IN = ('query', 'path', 'body', 'header')
PARAM_TYPES = ('string', 'integer', 'number', 'boolean', 'json', 'array')
SLUG_RE = re.compile(r'[a-z0-9]+(?:-[a-z0-9]+)*')
NAME_RE = re.compile(r'[A-Za-z0-9_.-]{1,80}')
MAX_BYTES = 25 * 1024 * 1024
class SpecError(ValueError):
pass
class CallError(ValueError):
pass
def host_of(url):
return (urllib.parse.urlsplit(url).hostname or '').lower()
def validate(spec):
"""List of problems (empty = valid). Strict on the shape so a bad spec never reaches an agent."""
errors = []
if not isinstance(spec, dict):
return ['spec must be an object']
slug = spec.get('slug')
if not isinstance(slug, str) or not SLUG_RE.fullmatch(slug) or len(slug) > 60:
errors.append('slug: lowercase letters, digits and dashes')
if not isinstance(spec.get('name'), str) or not spec['name'].strip():
errors.append('name required')
base = spec.get('base_url')
if not isinstance(base, str) or urllib.parse.urlsplit(base).scheme != 'https' or not host_of(base):
errors.append('base_url must be an https URL')
hosts = spec.get('allowed_hosts', [host_of(base)] if isinstance(base, str) else [])
if not isinstance(hosts, list) or not hosts or not all(isinstance(h, str) and h for h in hosts):
errors.append('allowed_hosts must be a non-empty list')
auth = spec.get('auth')
if not isinstance(auth, dict) or auth.get('type') not in AUTH_TYPES:
errors.append('auth.type must be one of ' + ', '.join(AUTH_TYPES))
else:
t = auth['type']
if t == 'api_key' and (auth.get('in') not in ('header', 'query') or not isinstance(auth.get('name'), str)):
errors.append('api_key auth needs in (header|query) and name')
if t == 'oauth2':
for k in ('authorize_url', 'token_url'):
if not isinstance(auth.get(k), str) or urllib.parse.urlsplit(auth[k]).scheme != 'https':
errors.append('oauth2 auth needs https ' + k)
if not isinstance(auth.get('scopes', []), list):
errors.append('oauth2 scopes must be a list')
creds = spec.get('credentials', [])
if not isinstance(creds, list) or not all(isinstance(c, dict) and isinstance(c.get('name'), str) and NAME_RE.fullmatch(c['name']) for c in creds):
errors.append('credentials must be a list of {name,label}')
cmds = spec.get('commands')
if not isinstance(cmds, dict) or not cmds:
errors.append('commands required')
cmds = {}
for name, c in cmds.items():
where = 'command ' + str(name)
if not isinstance(name, str) or not re.fullmatch(r'[a-z0-9][a-z0-9_.-]{0,60}', name) or name in ('test', 'help', 'auth', 'list'):
errors.append(where + ': name must be lowercase (test/help/auth/list are reserved)')
if not isinstance(c, dict):
errors.append(where + ': must be an object')
continue
if c.get('method', 'GET') not in METHODS:
errors.append(where + ': method')
path = c.get('path')
if not isinstance(path, str) or not path.startswith('/') or '://' in path:
errors.append(where + ': path must start with / (relative to base_url)')
path = ''
params = c.get('params', {})
if not isinstance(params, dict):
errors.append(where + ': params must be an object')
params = {}
for p, d in params.items():
if not isinstance(d, dict) or d.get('in', 'query') not in PARAM_IN or d.get('type', 'string') not in PARAM_TYPES:
errors.append('%s param %s: in (%s), type (%s)' % (where, p, '|'.join(PARAM_IN), '|'.join(PARAM_TYPES)))
for p in re.findall(r'\{([A-Za-z0-9_]+)\}', path):
if params.get(p, {}).get('in') != 'path':
errors.append('%s: {%s} in path needs a path param' % (where, p))
out = c.get('output', {'type': 'json'})
if not isinstance(out, dict) or out.get('type', 'json') not in ('json', 'text', 'file'):
errors.append(where + ': output.type json|text|file')
if c.get('body_type', 'json') not in ('json', 'form'):
errors.append(where + ': body_type json|form')
test = spec.get('test')
if not isinstance(test, dict) or test.get('command') not in cmds:
errors.append('test.command must name a command')
elif is_write(cmds[test['command']]):
errors.append('test.command must be read-only')
return errors
def is_write(cmd):
if 'write' in cmd:
return bool(cmd['write'])
return cmd.get('method', 'GET') not in ('GET', 'HEAD')
def _convert(value, kind):
if kind == 'integer':
return int(value)
if kind == 'number':
return float(value)
if kind == 'boolean':
return value if isinstance(value, bool) else str(value).lower() in ('1', 'true', 'yes', 'on')
if kind in ('json', 'array'):
return json.loads(value) if isinstance(value, str) else value
return value if isinstance(value, str) else (json.dumps(value) if isinstance(value, (dict, list)) else str(value))
def build_request(spec, command, params, cred, allow_write=False):
"""(method, url, headers, body_bytes, output) for one call. Raises CallError with a plain sentence."""
cmds = spec.get('commands', {})
if command not in cmds:
raise CallError('Unknown command %r. Commands: %s' % (command, ', '.join(sorted(cmds))))
c = cmds[command]
if is_write(c) and not allow_write:
raise CallError('"%s" changes things at %s, and writing is switched off for this connection. Ask the person to '
'allow writing in Mesh → Settings → Connections.' % (command, spec.get('name')))
params = dict(params or {})
declared = c.get('params', {})
unknown = sorted(set(params) - set(declared))
if unknown:
raise CallError('Unknown parameter(s) %s for %s. Known: %s' % (', '.join(unknown), command, ', '.join(sorted(declared)) or 'none'))
path, query, body, headers = c['path'], [], {}, {'User-Agent': 'osmike-int/1', 'Accept': 'application/json'}
headers.update(spec.get('headers') or {})
headers.update(c.get('headers') or {})
for name, d in declared.items():
if name not in params:
if 'default' in d:
params[name] = d['default']
elif d.get('required'):
raise CallError('%s needs --%s (%s)' % (command, name, d.get('description') or d.get('type', 'string')))
else:
continue
try:
value = _convert(params[name], d.get('type', 'string'))
except (ValueError, TypeError):
raise CallError('--%s must be %s' % (name, d.get('type'))) from None
where, key = d.get('in', 'query'), d.get('name', name)
if where == 'path':
path = path.replace('{' + name + '}', urllib.parse.quote(str(value), safe=''))
elif where == 'query':
for v in (value if isinstance(value, list) else [value]):
query.append((key, str(v).lower() if isinstance(v, bool) else str(v)))
elif where == 'header':
headers[key] = str(value)
else:
body[key] = value
if c.get('body') and isinstance(c['body'], dict): # fixed body fields
body = {**c['body'], **body}
auth = spec.get('auth', {})
t = auth.get('type')
cred = cred or {}
def need(field):
v = cred.get(field)
if not v:
raise CallError('The connection has no %s. Ask the person to connect %s again.' % (field, spec.get('name')))
return v
if t == 'api_key':
value = auth.get('prefix', '') + need(auth.get('field', 'api_key'))
if auth['in'] == 'header':
headers[auth['name']] = value
else:
query.append((auth['name'], value))
elif t == 'bearer':
headers['Authorization'] = 'Bearer ' + need(auth.get('field', 'token'))
elif t == 'basic':
user = need(auth.get('username_field', 'username'))
pw = cred.get(auth.get('password_field', 'password'), '')
headers['Authorization'] = 'Basic ' + base64.b64encode(('%s:%s' % (user, pw)).encode()).decode()
elif t == 'oauth2':
headers['Authorization'] = (cred.get('token_type') or 'Bearer').capitalize() + ' ' + need('access_token')
url = spec['base_url'].rstrip('/') + path
if query:
url += ('&' if '?' in url else '?') + urllib.parse.urlencode(query)
allowed = [h.lower() for h in spec.get('allowed_hosts') or [host_of(spec['base_url'])]]
if host_of(url) not in allowed:
raise CallError('Host %s is not allowed for %s' % (host_of(url), spec.get('name')))
data = None
method = c.get('method', 'GET')
if body and method not in ('GET', 'HEAD'):
if c.get('body_type', 'json') == 'form':
data = urllib.parse.urlencode({k: (json.dumps(v) if isinstance(v, (dict, list)) else v) for k, v in body.items()}).encode()
headers['Content-Type'] = 'application/x-www-form-urlencoded'
else:
data = json.dumps(body).encode()
headers['Content-Type'] = 'application/json'
elif body:
raise CallError('%s is a %s request and cannot take body parameters' % (command, method))
return method, url, headers, data, c.get('output', {'type': 'json'})
class _NoRedirectOffHost(urllib.request.HTTPRedirectHandler):
def __init__(self, allowed):
self.allowed = allowed
def redirect_request(self, req, fp, code, msg, headers, newurl):
if host_of(newurl) not in self.allowed:
raise CallError('The service redirected to another host (%s); refused' % host_of(newurl))
return super().redirect_request(req, fp, code, msg, headers, newurl)
def call(spec, command, params, cred, allow_write=False, timeout=60, opener=None):
"""Run one command. Returns {status, ok, content_type, json|text|bytes}. Never raises for HTTP errors."""
method, url, headers, data, out = build_request(spec, command, params, cred, allow_write)
allowed = [h.lower() for h in spec.get('allowed_hosts') or [host_of(spec['base_url'])]]
op = opener or urllib.request.build_opener(_NoRedirectOffHost(allowed))
req = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
resp = op.open(req, timeout=timeout)
status, ctype, raw = resp.status, resp.headers.get('Content-Type', ''), resp.read(MAX_BYTES + 1)
except urllib.error.HTTPError as e:
status, ctype, raw = e.code, e.headers.get('Content-Type', '') if e.headers else '', e.read(200_000)
except urllib.error.URLError as e:
raise CallError('Could not reach %s: %s' % (host_of(url), e.reason)) from None
if len(raw) > MAX_BYTES:
raise CallError('The answer is larger than 25 MB')
result = {'status': status, 'ok': 200 <= status < 300, 'content_type': ctype.split(';')[0].strip(), 'output': out.get('type', 'json')}
if out.get('type') == 'file' and result['ok'] and 'json' not in ctype:
result['bytes'], result['ext'] = raw, out.get('ext') or (ctype.split('/')[-1].split(';')[0] or 'bin')
return result
text = raw.decode('utf-8', 'replace')
try:
result['json'] = json.loads(text) if text.strip() else None
except ValueError:
result['text'] = text[:200_000]
return result
def secret_values(cred):
return [v for v in (cred or {}).values() if isinstance(v, str) and len(v) >= 6]
def scrub(text, cred):
"""Remove every credential value from text (answers, errors) before an agent or model sees it."""
for v in sorted(secret_values(cred), key=len, reverse=True):
text = text.replace(v, '[secret]')
return text
def describe(spec):
"""Short help for agents: commands, parameters, which ones write."""
lines = ['%s (osmike-int %s <command> --param value)' % (spec.get('name'), spec.get('slug'))]
for name, c in sorted(spec.get('commands', {}).items()):
ps = []
for p, d in c.get('params', {}).items():
ps.append(('--%s' % p) + ('' if d.get('required') else '?'))
lines.append(' %-22s %s%s %s' % (name, ' '.join(ps), ' [writes]' if is_write(c) else '', c.get('summary', '')))
return '\n'.join(lines)
# ---------------------------------------------------------------- OAuth 2 helpers (stdlib)
def authorize_url(spec, cred, redirect_uri, state, challenge=None):
a = spec['auth']
q = {'response_type': 'code', 'client_id': cred.get('client_id', ''), 'redirect_uri': redirect_uri, 'state': state}
if a.get('scopes'):
q['scope'] = a.get('scope_separator', ' ').join(a['scopes'])
if a.get('pkce', True) and challenge:
q.update(code_challenge=challenge, code_challenge_method='S256')
q.update(a.get('extra_authorize_params') or {})
return a['authorize_url'] + ('&' if '?' in a['authorize_url'] else '?') + urllib.parse.urlencode(q)
def token_request(spec, cred, form, opener=None, timeout=30):
"""POST to the token URL (code exchange or refresh). Returns the JSON token answer or raises CallError."""
a = spec['auth']
headers = {'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json', 'User-Agent': 'osmike-int/1'}
form = dict(form)
if a.get('client_auth', 'body') == 'basic':
headers['Authorization'] = 'Basic ' + base64.b64encode(('%s:%s' % (cred.get('client_id', ''), cred.get('client_secret', ''))).encode()).decode()
else:
form['client_id'] = cred.get('client_id', '')
if cred.get('client_secret'):
form['client_secret'] = cred['client_secret']
req = urllib.request.Request(a['token_url'], data=urllib.parse.urlencode(form).encode(), headers=headers, method='POST')
try:
raw = (opener or urllib.request.build_opener()).open(req, timeout=timeout).read(1_000_000)
except urllib.error.HTTPError as e:
raise CallError('The token request was refused (%s): %s' % (e.code, scrub(e.read(2000).decode('utf-8', 'replace'), cred)[:300])) from None
except urllib.error.URLError as e:
raise CallError('Could not reach the token URL: %s' % e.reason) from None
text = raw.decode('utf-8', 'replace')
try:
data = json.loads(text)
except ValueError:
data = dict(urllib.parse.parse_qsl(text)) # some providers answer form-encoded
if not data.get('access_token'):
raise CallError('The provider gave no access token: %s' % scrub(json.dumps(data), cred)[:300])
return data

204
osmike-int Executable file
View File

@ -0,0 +1,204 @@
#!/usr/bin/env python3
"""osmike-int: use any integrated service (Slack, ElevenLabs, ...) from an agent. Generic: one tool for all services.
osmike-int list services connected for you (and their state)
osmike-int catalog [word] every ready integration
osmike-int <service> what you can do with it (commands, parameters, which ones write)
osmike-int <service> test check that the connection works
osmike-int <service> <command> --param value [--param value ...] [--json '{"param": ...}'] [--save FILE]
The call runs on the OsMike server with the person's stored keys; you never see or need a key. Files (audio,
images, PDFs) come back as a link you can give the person. Commands marked [writes] change things at the service
and only work when the person has switched on "allow writing" for the connection in Mesh → Settings → Connections.
If a service is not connected for you: the person connects it by pasting its keys in the chat; Mesh does the rest.
Builder only (osmike-integrations):
osmike-int builder queue | claim <slug> [request_id] | fail <slug> "why"
osmike-int builder validate spec.json
osmike-int builder dryrun spec.json <command> --param value (shows the request; no network, no keys)
osmike-int builder trial spec.json <command> --param value (runs it with the waiting person's keys, server side)
osmike-int builder publish spec.json [--docs URL] [--notes TEXT] [--alias NAME ...]
"""
import json
import os
import sys
import urllib.error
import urllib.request
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
try:
import intengine
except ImportError: # the engine is only needed for builder validate/dryrun
intengine = None
def env():
api, key = os.environ.get('A2A_API', '').rstrip('/'), os.environ.get('A2A_AGENT_KEY', '')
if not api or not key:
die('A2A_API and A2A_AGENT_KEY must be set (they are in every agent).')
return api, key
def die(msg, code=1):
print(msg, file=sys.stderr)
sys.exit(code)
def http(method, path, body=None, timeout=120):
api, key = env()
req = urllib.request.Request(api + path, data=json.dumps(body).encode() if body is not None else None, method=method,
headers={'X-API-KEY': key, 'Content-Type': 'application/json'})
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.load(r)
except urllib.error.HTTPError as e:
try:
detail = json.load(e).get('detail')
except Exception:
detail = None
die('%s (HTTP %s)' % (detail if isinstance(detail, str) else json.dumps(detail) if detail else e.reason, e.code))
except urllib.error.URLError as e:
die('Could not reach OsMike: %s' % e.reason)
def parse_params(args):
params, save, i = {}, None, 0
while i < len(args):
a = args[i]
if a == '--json' and i + 1 < len(args):
try:
params.update(json.loads(args[i + 1]))
except ValueError:
die('--json must be a JSON object')
i += 2
elif a == '--save' and i + 1 < len(args):
save, i = args[i + 1], i + 2
elif a.startswith('--') and '=' in a:
k, v = a[2:].split('=', 1)
params[k.replace('-', '_')] = v
i += 1
elif a.startswith('--') and i + 1 < len(args) and not args[i + 1].startswith('--'):
params[a[2:].replace('-', '_')] = args[i + 1]
i += 2
elif a.startswith('--'):
params[a[2:].replace('-', '_')] = 'true'
i += 1
else:
die('Unexpected argument %r; parameters are --name value' % a)
return params, save
def show(res, save=None):
if res.get('file_url'):
print(json.dumps({'ok': res['ok'], 'file_url': res['file_url'], 'bytes': res.get('file_bytes')}, indent=1))
if save:
urllib.request.urlretrieve(res['file_url'], save)
print('saved to', save)
return
body = res.get('json') if res.get('json') is not None else res.get('text')
out = json.dumps(body, indent=1, ensure_ascii=False) if not isinstance(body, str) else body
if save:
Path(save).write_text(out)
print('saved to', save)
else:
print(out)
if not res.get('ok'):
print('(the service answered HTTP %s)' % res.get('status'), file=sys.stderr)
sys.exit(2)
def load_spec(path):
try:
return json.loads(Path(path).read_text())
except (OSError, ValueError) as e:
die('Cannot read spec %s: %s' % (path, e))
def builder(args):
if not args:
die(__doc__)
cmd, rest = args[0], args[1:]
if cmd == 'queue':
print(json.dumps(http('GET', '/v1/runs/integrations/queue'), indent=1))
elif cmd == 'claim' and rest:
print(json.dumps(http('POST', '/v1/runs/integrations/%s/claim' % rest[0], {'request_id': int(rest[1]) if len(rest) > 1 else None}), indent=1))
elif cmd == 'fail' and len(rest) >= 2:
print(json.dumps(http('POST', '/v1/runs/integrations/%s/fail' % rest[0], {'error': ' '.join(rest[1:])}), indent=1))
elif cmd == 'validate' and rest:
if not intengine:
die('intengine.py missing next to osmike-int')
errors = intengine.validate(load_spec(rest[0]))
print('valid' if not errors else '\n'.join('- ' + e for e in errors))
sys.exit(1 if errors else 0)
elif cmd == 'dryrun' and len(rest) >= 2:
spec = load_spec(rest[0])
params, _ = parse_params(rest[2:])
fake = {f.get('name'): 'SECRET-' + f.get('name', 'x').upper() for f in spec.get('credentials', [])}
fake.update(access_token='SECRET-ACCESS-TOKEN', token='SECRET-TOKEN', api_key=fake.get('api_key', 'SECRET-API-KEY'))
try:
method, url, headers, data, out = intengine.build_request(spec, rest[1], params, fake, allow_write=True)
except intengine.CallError as e:
die(str(e))
print(json.dumps({'method': method, 'url': url, 'headers': headers, 'body': data.decode() if data else None,
'output': out, 'writes': intengine.is_write(spec['commands'][rest[1]])}, indent=1))
elif cmd == 'trial' and len(rest) >= 2:
spec = load_spec(rest[0])
params, _ = parse_params(rest[2:])
print(json.dumps(http('POST', '/v1/runs/integrations/%s/trial' % spec.get('slug'), {'spec': spec, 'command': rest[1], 'params': params}), indent=1, ensure_ascii=False))
elif cmd == 'publish' and rest:
spec = load_spec(rest[0])
opts, aliases, i = {}, [], 1
while i < len(rest):
if rest[i] in ('--docs', '--notes') and i + 1 < len(rest):
opts[rest[i][2:]] = rest[i + 1]
i += 2
elif rest[i] == '--alias' and i + 1 < len(rest):
aliases.append(rest[i + 1])
i += 2
else:
die('publish spec.json [--docs URL] [--notes TEXT] [--alias NAME]')
print(json.dumps(http('POST', '/v1/runs/integrations/%s/publish' % spec.get('slug'),
{'spec': spec, 'docs_url': opts.get('docs'), 'notes': opts.get('notes'), 'aliases': aliases}), indent=1))
else:
die(__doc__)
def main(argv):
if not argv or argv[0] in ('-h', '--help', 'help'):
print(__doc__)
return
if argv[0] == 'builder':
return builder(argv[1:])
if argv[0] == 'list':
rows = http('GET', '/v1/runs/integrations/mine')['connections']
if not rows:
print('No services are connected for you yet. A person connects one by pasting its keys in the chat.')
for r in rows:
print('%-20s %-10s writing %s (for %s)' % (r['slug'], r['status'], 'on' if r['allow_write'] else 'off', r['person']))
return
if argv[0] == 'catalog':
q = ('?q=' + urllib.request.quote(argv[1])) if len(argv) > 1 else ''
for r in http('GET', '/v1/runs/integrations/catalog' + q)['integrations']:
print('%-24s %s' % (r['slug'], r['name']))
return
slug, rest = argv[0], argv[1:]
if not rest or rest[0] in ('-h', '--help', 'help'):
spec = http('GET', '/v1/runs/integrations/%s/spec' % slug)['spec']
if intengine:
print(intengine.describe(spec))
else:
print(json.dumps({k: v.get('summary') for k, v in spec['commands'].items()}, indent=1))
return
if rest[0] == 'test':
spec = http('GET', '/v1/runs/integrations/%s/spec' % slug)['spec']
t = spec['test']
res = http('POST', '/v1/runs/integrations/%s/call' % slug, {'command': t['command'], 'params': t.get('params') or {}})
print('%s works (HTTP %s)' % (spec['name'], res['status']) if res['ok'] else '%s did not work (HTTP %s)' % (spec['name'], res['status']))
sys.exit(0 if res['ok'] else 2)
params, save = parse_params(rest[1:])
show(http('POST', '/v1/runs/integrations/%s/call' % slug, {'command': rest[0], 'params': params}), save)
if __name__ == '__main__':
main(sys.argv[1:])