Skip to content

Commit fb9c396

Browse files
committed
refactor(e2e): run MCP conformance runner in a container
The upstream MCP conformance runner is untrusted node, so run it in a plain Docker container on the e2e network instead of on the host. Unlike an OpenShell sandbox (egress-only, so an inbound test server is unreachable), the container has a routable address the client sandbox can reach, fixing the scenario connection timeout. The runner holds no gateway credentials: its client command posts the test server URL back to a host-side bridge that runs the real MCP client in an OpenShell sandbox and returns the result. That callback is the trust boundary that keeps gateway access off the untrusted runner. - host-bridge.py: host HTTP server that launches the client in a sandbox - runner-shim.mjs: the runner's client command; calls back to the host bridge - render-policy.py: policy renderer shared with client-through-openshell.sh - build dist in the client image and run `node dist/index.js` so scenarios do not pay the tsx transpile cost on every run - drop the host Node.js/npm build (and the CI Node.js setup step) and the DROP_DOCKER_AUTH knob, both unnecessary now that node runs in a container Signed-off-by: Kris Hicks <khicks@nvidia.com>
1 parent b92cd00 commit fb9c396

4 files changed

Lines changed: 281 additions & 7 deletions

File tree

‎e2e/mcp-conformance.sh‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -320,17 +320,26 @@ run_scenarios_in_runner_container() {
320320
scenario_list="$(scenario_list_for_args "$@")"
321321
if [ -z "${scenario_list}" ]; then
322322
echo "ERROR: no MCP conformance scenarios resolved." >&2
323-
exit 2
323+
return 2
324324
fi
325325

326+
bridge_port="$(e2e_pick_port)"
327+
create_client_sandbox || return 1
328+
start_host_bridge "${bridge_port}" || return 1
329+
create_runner_container || return 1
330+
331+
bridge_host="$(host_bridge_hostname)"
332+
echo "MCP conformance host bridge callback: http://${bridge_host}:${bridge_port}/run" >&2
333+
326334
for scenario in ${scenario_list}; do
327335
echo "=== MCP conformance: ${scenario} ==="
328-
if node "${CONFORMANCE_DIR}/dist/index.js" client \
329-
--command "bash e2e/mcp-conformance/client-through-openshell.sh" \
330-
--scenario "${scenario}" \
331-
--spec-version "${SPEC_VERSION}" \
332-
--expected-failures "${ROOT}/e2e/mcp-conformance/expected-failures.yml" \
333-
--timeout "${TIMEOUT_MS}"; then
336+
# shellcheck disable=SC2016
337+
if docker exec \
338+
--env "MCP_CONFORMANCE_HOST_BRIDGE_URL=http://${bridge_host}:${bridge_port}/run" \
339+
"${RUNNER_CONTAINER}" \
340+
sh -c 'cd /opt/mcp-conformance && exec node dist/index.js client --command "node /tmp/openshell-mcp-runner-shim.mjs" --scenario "$1" --spec-version "$2" --expected-failures "$3" --timeout "$4"' \
341+
sh "${scenario}" "${SPEC_VERSION}" "/tmp/expected-failures.yml" "${TIMEOUT_MS}" \
342+
</dev/null; then
334343
passed+=("${scenario}")
335344
else
336345
failed+=("${scenario}")

‎e2e/mcp-conformance/host-bridge.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env python3
2+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
"""Host bridge for the MCP conformance runner.
6+
7+
The conformance runner runs in an isolated container and posts the URL of its
8+
MCP test server to this bridge. The bridge runs the real MCP client inside an
9+
OpenShell sandbox (via client-through-openshell.sh) and returns its result, so
10+
the untrusted runner never needs gateway credentials.
11+
12+
Usage: host-bridge.py <port> <repo-root> <log-path>
13+
"""
14+
15+
import json
16+
import os
17+
import subprocess
18+
import sys
19+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
20+
from pathlib import Path
21+
22+
PORT = int(sys.argv[1])
23+
ROOT = Path(sys.argv[2])
24+
LOG_PATH = Path(sys.argv[3])
25+
TIMEOUT = int(os.environ.get("OPENSHELL_MCP_CONFORMANCE_CLIENT_TIMEOUT_SECONDS", "120")) + 30
26+
27+
28+
def log(message: str) -> None:
29+
with LOG_PATH.open("a", encoding="utf-8") as fh:
30+
fh.write(message + "\n")
31+
32+
33+
class Handler(BaseHTTPRequestHandler):
34+
def do_POST(self) -> None:
35+
if self.path != "/run":
36+
self.send_response(404)
37+
self.end_headers()
38+
return
39+
length = int(self.headers.get("content-length", "0"))
40+
payload = json.loads(self.rfile.read(length))
41+
server_url = payload["server_url"]
42+
env = os.environ.copy()
43+
env.update(payload.get("env", {}))
44+
env.pop("MISE_GITHUB_TOKEN", None)
45+
log(f"running client for {server_url}")
46+
try:
47+
result = subprocess.run(
48+
["bash", "e2e/mcp-conformance/client-through-openshell.sh", server_url],
49+
cwd=ROOT,
50+
env=env,
51+
stdin=subprocess.DEVNULL,
52+
stdout=subprocess.PIPE,
53+
stderr=subprocess.PIPE,
54+
text=True,
55+
timeout=TIMEOUT,
56+
)
57+
log(f"client exited {result.returncode} for {server_url}")
58+
body = {
59+
"exit_code": result.returncode,
60+
"stdout": result.stdout,
61+
"stderr": result.stderr,
62+
}
63+
except subprocess.TimeoutExpired as err:
64+
body = {
65+
"exit_code": 124,
66+
"stdout": err.stdout or "",
67+
"stderr": (err.stderr or "") + f"\nhost bridge timed out after {TIMEOUT}s\n",
68+
}
69+
encoded = json.dumps(body).encode("utf-8")
70+
self.send_response(200)
71+
self.send_header("content-type", "application/json")
72+
self.send_header("content-length", str(len(encoded)))
73+
self.end_headers()
74+
self.wfile.write(encoded)
75+
76+
def log_message(self, fmt: str, *args: object) -> None:
77+
log(fmt % args)
78+
79+
80+
def main() -> None:
81+
server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler)
82+
log(f"host bridge listening on {PORT}")
83+
server.serve_forever()
84+
85+
86+
if __name__ == "__main__":
87+
main()
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/usr/bin/env python3
2+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
"""Render the OpenShell client policy for a conformance server URL.
6+
7+
Parses the server URL into host/port/path, substitutes them into
8+
policy-template.yaml, writes the rendered policy, and prints the (possibly
9+
rewritten) URL the client should connect to. Used both to seed the reusable
10+
client sandbox with a placeholder policy and to install the per-scenario policy
11+
in client-through-openshell.sh.
12+
13+
Usage: render-policy.py <server-url> <policy-file> <policy-template>
14+
"""
15+
16+
import json
17+
import string
18+
import sys
19+
from pathlib import Path
20+
from urllib.parse import urlparse, urlunparse
21+
22+
raw_url, policy_file, policy_template = sys.argv[1:4]
23+
parsed = urlparse(raw_url)
24+
25+
if parsed.scheme not in ("http", "https"):
26+
raise SystemExit(f"unsupported conformance server URL scheme: {parsed.scheme!r}")
27+
28+
host = parsed.hostname
29+
if not host:
30+
raise SystemExit(f"conformance server URL is missing a host: {raw_url}")
31+
32+
target_host = "host.openshell.internal" if host in {"localhost", "127.0.0.1", "::1"} else host
33+
port = parsed.port or (443 if parsed.scheme == "https" else 80)
34+
path = parsed.path or "/"
35+
netloc_host = f"[{target_host}]" if ":" in target_host and not target_host.startswith("[") else target_host
36+
netloc = f"{netloc_host}:{port}"
37+
rewritten = urlunparse((parsed.scheme, netloc, path, parsed.params, parsed.query, parsed.fragment))
38+
39+
template = string.Template(Path(policy_template).read_text(encoding="utf-8"))
40+
policy = template.substitute(
41+
host_spec=f"host: {json.dumps(target_host)}",
42+
port_spec=f" port: {port}",
43+
path=json.dumps(path),
44+
)
45+
Path(policy_file).write_text(policy, encoding="utf-8")
46+
47+
print(rewritten)
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
// Client command for the upstream MCP conformance runner, executed inside the
5+
// runner container.
6+
//
7+
// Why this indirection exists: the conformance runner runs untrusted upstream
8+
// node off the host, so it deliberately has no openshell binary and no gateway
9+
// credentials. But the MCP client under test must run inside an OpenShell
10+
// sandbox so its traffic crosses the policy-enforced proxy (the whole point of
11+
// the e2e). This script bridges that gap without widening the runner's
12+
// privileges: instead of running the MCP client itself, it posts the test
13+
// server URL to the host bridge, which holds the gateway credentials and runs
14+
// the real client in a sandbox, then returns its result. The runner can only
15+
// ask "run a client against this URL" — it cannot touch the gateway control
16+
// plane.
17+
18+
import http from 'node:http';
19+
import os from 'node:os';
20+
import process from 'node:process';
21+
22+
// Resolve the routable IPv4 address of the runner container on the e2e Docker
23+
// network. The conformance runner reports its test server URL as localhost; the
24+
// client sandbox connects from a different container, so localhost must be
25+
// rewritten to this container's network address.
26+
function runnerAddress() {
27+
for (const addrs of Object.values(os.networkInterfaces())) {
28+
for (const addr of addrs ?? []) {
29+
if (addr.family === 'IPv4' && !addr.internal) {
30+
return addr.address;
31+
}
32+
}
33+
}
34+
throw new Error('failed to resolve runner container IPv4 address');
35+
}
36+
37+
function rewriteServerUrl(rawUrl) {
38+
const url = new URL(rawUrl);
39+
if (['localhost', '127.0.0.1', '[::1]', '::1'].includes(url.hostname)) {
40+
url.hostname = runnerAddress();
41+
}
42+
return url.toString();
43+
}
44+
45+
const bridgeUrl = process.env.MCP_CONFORMANCE_HOST_BRIDGE_URL;
46+
if (!bridgeUrl) {
47+
throw new Error('MCP_CONFORMANCE_HOST_BRIDGE_URL is required');
48+
}
49+
const parsedBridgeUrl = new URL(bridgeUrl);
50+
if (parsedBridgeUrl.protocol !== 'http:') {
51+
throw new Error(`bridge URL must use http:, got ${parsedBridgeUrl.protocol}`);
52+
}
53+
const serverUrl = process.argv[2];
54+
if (!serverUrl) {
55+
throw new Error('usage: node runner-shim.mjs <server-url>');
56+
}
57+
58+
// POST the rewritten server URL to the host bridge, which runs the real MCP
59+
// client inside an OpenShell sandbox and returns its result. The runner is a
60+
// plain container with ordinary egress, so this is a direct HTTP call.
61+
function postJson(url, payload) {
62+
const body = Buffer.from(JSON.stringify(payload), 'utf8');
63+
const timeoutMs = Number.parseInt(process.env.MCP_CONFORMANCE_HOST_BRIDGE_TIMEOUT_MS ?? '600000', 10);
64+
65+
function snippet(raw) {
66+
return raw.length > 4096 ? `${raw.slice(0, 4096)}...` : raw;
67+
}
68+
69+
return new Promise((resolve, reject) => {
70+
const request = http.request({
71+
hostname: url.hostname,
72+
port: url.port || 80,
73+
path: `${url.pathname}${url.search}`,
74+
method: 'POST',
75+
headers: {
76+
'content-type': 'application/json',
77+
'content-length': body.length,
78+
},
79+
agent: false,
80+
}, (response) => {
81+
const chunks = [];
82+
response.on('data', (chunk) => chunks.push(chunk));
83+
response.on('end', () => {
84+
const raw = Buffer.concat(chunks).toString('utf8');
85+
const statusCode = response.statusCode ?? 0;
86+
try {
87+
const parsed = JSON.parse(raw);
88+
if (statusCode < 200 || statusCode >= 300) {
89+
reject(new Error(`bridge callback failed (HTTP ${statusCode}): ${snippet(raw)}`));
90+
return;
91+
}
92+
resolve(parsed);
93+
} catch (error) {
94+
reject(new Error(`bridge returned invalid JSON (HTTP ${statusCode}): ${snippet(raw) || error.message}`));
95+
}
96+
});
97+
});
98+
request.on('error', reject);
99+
request.setTimeout(timeoutMs, () => {
100+
request.destroy(new Error(`bridge callback timed out after ${timeoutMs}ms to ${url.toString()}`));
101+
});
102+
request.end(body);
103+
});
104+
}
105+
106+
const forwardedEnv = {};
107+
for (const name of [
108+
'MCP_CONFORMANCE_SCENARIO',
109+
'MCP_CONFORMANCE_CONTEXT',
110+
'MCP_CONFORMANCE_PROTOCOL_VERSION',
111+
]) {
112+
if (process.env[name] !== undefined) {
113+
forwardedEnv[name] = process.env[name];
114+
}
115+
}
116+
117+
const body = await postJson(parsedBridgeUrl, {
118+
server_url: rewriteServerUrl(serverUrl),
119+
env: forwardedEnv,
120+
});
121+
if (typeof body.exit_code !== 'number') {
122+
const raw = JSON.stringify(body);
123+
throw new Error(`bridge returned JSON without numeric exit_code: ${raw.length > 4096 ? `${raw.slice(0, 4096)}...` : raw}`);
124+
}
125+
if (body.stdout) {
126+
process.stdout.write(body.stdout);
127+
}
128+
if (body.stderr) {
129+
process.stderr.write(body.stderr);
130+
}
131+
process.exit(body.exit_code);

0 commit comments

Comments
 (0)