Plugin runtime v2 (#62)

2026-03-18 07:23:51 +01:00 · 2024-08-08 21:30:59 -07:00
parent f967820f12
commit 063e6cf00c
64 changed files with 1539 additions and 705 deletions
--- a/plugin-runtime/package-lock.json
+++ b/plugin-runtime/package-lock.json
@@ -6,11 +6,13 @@
    "": {
      "name": "@yaak/plugin-runtime",
      "dependencies": {
+        "intercept-stdout": "^0.1.2",
        "long": "^5.2.3",
        "nice-grpc": "^2.1.9",
        "protobufjs": "^7.3.2"
      },
      "devDependencies": {
+        "@types/intercept-stdout": "^0.1.3",
        "grpc-tools": "^1.12.4",
        "nodemon": "^3.1.4",
        "npm-run-all": "^4.1.5",
@@ -192,6 +194,12 @@
      "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==",
      "dev": true
    },
+    "node_modules/@types/intercept-stdout": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/@types/intercept-stdout/-/intercept-stdout-0.1.3.tgz",
+      "integrity": "sha512-5qWSvqohM5rRKsF58LBWJeyu+lUlZwYKSnTcnXGfvFyMYIjvhpfniQRJNiyE/Gcru3jwVr2pHedsKTGLtzZqNA==",
+      "dev": true
+    },
    "node_modules/@types/node": {
      "version": "20.14.7",
      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.7.tgz",
@@ -1304,6 +1312,14 @@
      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
      "dev": true
    },
+    "node_modules/intercept-stdout": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/intercept-stdout/-/intercept-stdout-0.1.2.tgz",
+      "integrity": "sha512-Umb41Ryp5FzLurfCRAWx+jjNAk8jsw2RTk2XPIwus+86h/Y2Eb4DfOWof/mZ6FBww8SoO45rJSlg25054/Di9w==",
+      "dependencies": {
+        "lodash.toarray": "^3.0.0"
+      }
+    },
    "node_modules/internal-slot": {
      "version": "1.0.7",
      "resolved": "https://registry.npmjs.org/internal-slot/-/internal-slot-1.0.7.tgz",
@@ -1644,11 +1660,56 @@
        "node": ">=4"
      }
    },
+    "node_modules/lodash._arraycopy": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/lodash._arraycopy/-/lodash._arraycopy-3.0.0.tgz",
+      "integrity": "sha512-RHShTDnPKP7aWxlvXKiDT6IX2jCs6YZLCtNhOru/OX2Q/tzX295vVBK5oX1ECtN+2r86S0Ogy8ykP1sgCZAN0A=="
+    },
+    "node_modules/lodash._basevalues": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/lodash._basevalues/-/lodash._basevalues-3.0.0.tgz",
+      "integrity": "sha512-H94wl5P13uEqlCg7OcNNhMQ8KvWSIyqXzOPusRgHC9DK3o54P6P3xtbXlVbRABG4q5gSmp7EDdJ0MSuW9HX6Mg=="
+    },
+    "node_modules/lodash._getnative": {
+      "version": "3.9.1",
+      "resolved": "https://registry.npmjs.org/lodash._getnative/-/lodash._getnative-3.9.1.tgz",
+      "integrity": "sha512-RrL9VxMEPyDMHOd9uFbvMe8X55X16/cGM5IgOKgRElQZutpX89iS6vwl64duTV1/16w5JY7tuFNXqoekmh1EmA=="
+    },
    "node_modules/lodash.camelcase": {
      "version": "4.3.0",
      "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
      "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
    },
+    "node_modules/lodash.isarguments": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
+      "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="
+    },
+    "node_modules/lodash.isarray": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/lodash.isarray/-/lodash.isarray-3.0.4.tgz",
+      "integrity": "sha512-JwObCrNJuT0Nnbuecmqr5DgtuBppuCvGD9lxjFpAzwnVtdGoDQ1zig+5W8k5/6Gcn0gZ3936HDAlGd28i7sOGQ=="
+    },
+    "node_modules/lodash.keys": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-3.1.2.tgz",
+      "integrity": "sha512-CuBsapFjcubOGMn3VD+24HOAPxM79tH+V6ivJL3CHYjtrawauDJHUk//Yew9Hvc6e9rbCrURGk8z6PC+8WJBfQ==",
+      "dependencies": {
+        "lodash._getnative": "^3.0.0",
+        "lodash.isarguments": "^3.0.0",
+        "lodash.isarray": "^3.0.0"
+      }
+    },
+    "node_modules/lodash.toarray": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/lodash.toarray/-/lodash.toarray-3.0.2.tgz",
+      "integrity": "sha512-ptkjUqvuHjTuMJJxiktJpZhxM5l60bEkfntJx+NFzdQd1bZVxfpTF1bhFYFqBrT4F0wZ1qx9KbVmHJV3Rfc7Tw==",
+      "dependencies": {
+        "lodash._arraycopy": "^3.0.0",
+        "lodash._basevalues": "^3.0.0",
+        "lodash.keys": "^3.0.0"
+      }
+    },
    "node_modules/long": {
      "version": "5.2.3",
      "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz",
--- a/plugin-runtime/package.json
+++ b/plugin-runtime/package.json
@@ -8,11 +8,13 @@
    "build:proto": "grpc_tools_node_protoc --ts_proto_out=src/gen --ts_proto_opt=outputServices=nice-grpc,outputServices=generic-definitions,useExactTypes=false --proto_path=../proto ../proto/plugins/*.proto"
  },
  "dependencies": {
+    "intercept-stdout": "^0.1.2",
    "long": "^5.2.3",
    "nice-grpc": "^2.1.9",
    "protobufjs": "^7.3.2"
  },
  "devDependencies": {
+    "@types/intercept-stdout": "^0.1.3",
    "grpc-tools": "^1.12.4",
    "nodemon": "^3.1.4",
    "npm-run-all": "^4.1.5",
--- a/plugin-runtime/src/EventChannel.ts
+++ b/plugin-runtime/src/EventChannel.ts
@@ -0,0 +1,21 @@
+import { InternalEvent } from '@yaakapp/api';
+import EventEmitter from 'node:events';
+import { EventStreamEvent } from './gen/plugins/runtime';
+
+export class EventChannel {
+  emitter: EventEmitter = new EventEmitter();
+
+  emit(e: InternalEvent) {
+    this.emitter.emit('__plugin_event__', { event: JSON.stringify(e) });
+  }
+
+  async *listen(): AsyncGenerator<EventStreamEvent> {
+    while (true) {
+      yield new Promise<EventStreamEvent>((resolve) => {
+        this.emitter.once('__plugin_event__', (event: EventStreamEvent) => {
+          resolve(event);
+        });
+      });
+    }
+  }
+}
--- a/plugin-runtime/src/PluginHandle.ts
+++ b/plugin-runtime/src/PluginHandle.ts
@@ -1,81 +1,31 @@
-import { randomUUID } from 'node:crypto';
+import { InternalEvent } from '@yaakapp/api';
 import path from 'node:path';
 import { Worker } from 'node:worker_threads';
-import { PluginInfo } from './plugins';
-
-export interface ParentToWorkerEvent<T = any> {
-  callbackId: string;
-  name: string;
-  payload: T;
-}
-
-export type WorkerToParentSuccessEvent<T> = {
-  callbackId: string;
-  payload: T;
-};
-
-export type WorkerToParentErrorEvent = {
-  callbackId: string;
-  error: string;
-};
-
-export type WorkerToParentEvent<T = any> = WorkerToParentErrorEvent | WorkerToParentSuccessEvent<T>;
+import { EventChannel } from './EventChannel';

 export class PluginHandle {
-  readonly pluginDir: string;
  readonly #worker: Worker;

-  constructor(pluginDir: string) {
-    this.pluginDir = pluginDir;
-
-    const workerPath = path.join(__dirname, 'index.worker.cjs');
+  constructor(
+    readonly pluginDir: string,
+    readonly pluginRefId: string,
+    readonly events: EventChannel,
+  ) {
+    const workerPath = process.env.YAAK_WORKER_PATH ?? path.join(__dirname, 'index.worker.cjs');
    this.#worker = new Worker(workerPath, {
      workerData: {
-        pluginDir: this.pluginDir,
+        pluginDir,
+        pluginRefId,
      },
    });

+    this.#worker.on('message', (e) => this.events.emit(e));
    this.#worker.on('error', this.#handleError.bind(this));
    this.#worker.on('exit', this.#handleExit.bind(this));
  }

-  async getInfo(): Promise<PluginInfo> {
-    return this.#callPlugin('info', null);
-  }
-
-  async runResponseFilter({ filter, body }: { filter: string; body: string }): Promise<string> {
-    return this.#callPlugin('run-filter', { filter, body });
-  }
-
-  async runExport(request: any): Promise<string> {
-    return this.#callPlugin('run-export', request);
-  }
-
-  async runImport(data: string): Promise<string> {
-    const result = await this.#callPlugin('run-import', data);
-
-    // Plugin returns object, but we convert to string
-    return JSON.stringify(result, null, 2);
-  }
-
-  #callPlugin<P, R>(name: string, payload: P): Promise<R> {
-    const callbackId = `cb_${randomUUID().replaceAll('-', '')}`;
-    return new Promise((resolve, reject) => {
-      const cb = (e: WorkerToParentEvent<R>) => {
-        if (e.callbackId !== callbackId) return;
-
-        if ('error' in e) {
-          reject(e.error);
-        } else {
-          resolve(e.payload as R);
-        }
-
-        this.#worker.removeListener('message', cb);
-      };
-
-      this.#worker.addListener('message', cb);
-      this.#worker.postMessage({ callbackId, name, payload });
-    });
+  sendToWorker(event: InternalEvent) {
+    this.#worker.postMessage(event);
  }

  async #handleError(err: Error) {
--- a/plugin-runtime/src/PluginManager.ts
+++ b/plugin-runtime/src/PluginManager.ts
@@ -1,44 +0,0 @@
-import { PluginHandle } from './PluginHandle';
-import { loadPlugins, PluginInfo } from './plugins';
-
-export class PluginManager {
-  #handles: PluginHandle[] | null = null;
-  static #instance: PluginManager | null = null;
-
-  public static instance(): PluginManager {
-    if (PluginManager.#instance == null) {
-      PluginManager.#instance = new PluginManager();
-      PluginManager.#instance.plugins(); // Trigger workers to boot, as it takes a few seconds
-    }
-    return PluginManager.#instance;
-  }
-
-  async plugins(): Promise<PluginHandle[]> {
-    this.#handles = this.#handles ?? loadPlugins();
-    return this.#handles;
-  }
-
-  async #pluginsWithInfo(): Promise<{ plugin: PluginHandle; info: PluginInfo }[]> {
-    const plugins = await this.plugins();
-    return Promise.all(plugins.map(async (plugin) => ({ plugin, info: await plugin.getInfo() })));
-  }
-
-  async pluginsWith(capability: PluginInfo['capabilities'][0]): Promise<PluginHandle[]> {
-    return (await this.#pluginsWithInfo())
-      .filter((v) => v.info.capabilities.includes(capability))
-      .map((v) => v.plugin);
-  }
-
-  async plugin(name: string): Promise<PluginHandle | null> {
-    return (await this.#pluginsWithInfo()).find((v) => v.info.name === name)?.plugin ?? null;
-  }
-
-  async pluginOrThrow(name: string): Promise<PluginHandle> {
-    const plugin = await this.plugin(name);
-    if (plugin == null) {
-      throw new Error(`Failed to find plugin by ${name}`);
-    }
-
-    return plugin;
-  }
-}
--- a/plugin-runtime/src/gen/plugins/runtime.ts
+++ b/plugin-runtime/src/gen/plugins/runtime.ts
@@ -33,6 +33,10 @@ export interface HookExportRequest {
  request: string;
 }

+export interface EventStreamEvent {
+  event: string;
+}
+
 function createBasePluginInfo(): PluginInfo {
  return { plugin: "" };
 }
@@ -369,54 +373,91 @@ export const HookExportRequest = {
  },
 };

+function createBaseEventStreamEvent(): EventStreamEvent {
+  return { event: "" };
+}
+
+export const EventStreamEvent = {
+  encode(message: EventStreamEvent, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.event !== "") {
+      writer.uint32(10).string(message.event);
+    }
+    return writer;
+  },
+
+  decode(input: _m0.Reader | Uint8Array, length?: number): EventStreamEvent {
+    const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input);
+    let end = length === undefined ? reader.len : reader.pos + length;
+    const message = createBaseEventStreamEvent();
+    while (reader.pos < end) {
+      const tag = reader.uint32();
+      switch (tag >>> 3) {
+        case 1:
+          if (tag !== 10) {
+            break;
+          }
+
+          message.event = reader.string();
+          continue;
+      }
+      if ((tag & 7) === 4 || tag === 0) {
+        break;
+      }
+      reader.skipType(tag & 7);
+    }
+    return message;
+  },
+
+  fromJSON(object: any): EventStreamEvent {
+    return { event: isSet(object.event) ? globalThis.String(object.event) : "" };
+  },
+
+  toJSON(message: EventStreamEvent): unknown {
+    const obj: any = {};
+    if (message.event !== "") {
+      obj.event = message.event;
+    }
+    return obj;
+  },
+
+  create(base?: DeepPartial<EventStreamEvent>): EventStreamEvent {
+    return EventStreamEvent.fromPartial(base ?? {});
+  },
+  fromPartial(object: DeepPartial<EventStreamEvent>): EventStreamEvent {
+    const message = createBaseEventStreamEvent();
+    message.event = object.event ?? "";
+    return message;
+  },
+};
+
 export type PluginRuntimeDefinition = typeof PluginRuntimeDefinition;
 export const PluginRuntimeDefinition = {
  name: "PluginRuntime",
  fullName: "yaak.plugins.runtime.PluginRuntime",
  methods: {
-    hookImport: {
-      name: "hookImport",
-      requestType: HookImportRequest,
-      requestStream: false,
-      responseType: HookResponse,
-      responseStream: false,
-      options: {},
-    },
-    hookExport: {
-      name: "hookExport",
-      requestType: HookExportRequest,
-      requestStream: false,
-      responseType: HookResponse,
-      responseStream: false,
-      options: {},
-    },
-    hookResponseFilter: {
-      name: "hookResponseFilter",
-      requestType: HookResponseFilterRequest,
-      requestStream: false,
-      responseType: HookResponse,
-      responseStream: false,
+    eventStream: {
+      name: "EventStream",
+      requestType: EventStreamEvent,
+      requestStream: true,
+      responseType: EventStreamEvent,
+      responseStream: true,
      options: {},
    },
  },
 } as const;

 export interface PluginRuntimeServiceImplementation<CallContextExt = {}> {
-  hookImport(request: HookImportRequest, context: CallContext & CallContextExt): Promise<DeepPartial<HookResponse>>;
-  hookExport(request: HookExportRequest, context: CallContext & CallContextExt): Promise<DeepPartial<HookResponse>>;
-  hookResponseFilter(
-    request: HookResponseFilterRequest,
+  eventStream(
+    request: AsyncIterable<EventStreamEvent>,
    context: CallContext & CallContextExt,
-  ): Promise<DeepPartial<HookResponse>>;
+  ): ServerStreamingMethodResult<DeepPartial<EventStreamEvent>>;
 }

 export interface PluginRuntimeClient<CallOptionsExt = {}> {
-  hookImport(request: DeepPartial<HookImportRequest>, options?: CallOptions & CallOptionsExt): Promise<HookResponse>;
-  hookExport(request: DeepPartial<HookExportRequest>, options?: CallOptions & CallOptionsExt): Promise<HookResponse>;
-  hookResponseFilter(
-    request: DeepPartial<HookResponseFilterRequest>,
+  eventStream(
+    request: AsyncIterable<DeepPartial<EventStreamEvent>>,
    options?: CallOptions & CallOptionsExt,
-  ): Promise<HookResponse>;
+  ): AsyncIterable<EventStreamEvent>;
 }

 type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined;
@@ -430,3 +471,5 @@ export type DeepPartial<T> = T extends Builtin ? T
 function isSet(value: any): boolean {
  return value !== null && value !== undefined;
 }
+
+export type ServerStreamingMethodResult<Response> = { [Symbol.asyncIterator](): AsyncIterator<Response, void> };
--- a/plugin-runtime/src/index.ts
+++ b/plugin-runtime/src/index.ts
@@ -1,87 +1,42 @@
-import { isAbortError } from 'abort-controller-x';
-import { createServer, ServerError, ServerMiddlewareCall, Status } from 'nice-grpc';
-import { CallContext } from 'nice-grpc-common';
-import * as fs from 'node:fs';
-import {
-  DeepPartial,
-  HookExportRequest,
-  HookImportRequest,
-  HookResponse,
-  HookResponseFilterRequest,
-  PluginRuntimeDefinition,
-  PluginRuntimeServiceImplementation,
-} from './gen/plugins/runtime';
-import { PluginManager } from './PluginManager';
+import { InternalEvent } from '@yaakapp/api';
+import { createChannel, createClient, Status } from 'nice-grpc';
+import { EventChannel } from './EventChannel';
+import { PluginRuntimeClient, PluginRuntimeDefinition } from './gen/plugins/runtime';
+import { PluginHandle } from './PluginHandle';

-class PluginRuntimeService implements PluginRuntimeServiceImplementation {
-  #manager: PluginManager;
+const port = process.env.PORT || '50051';

-  constructor() {
-    this.#manager = PluginManager.instance();
-  }
+const channel = createChannel(`localhost:${port}`);
+const client: PluginRuntimeClient = createClient(PluginRuntimeDefinition, channel);

-  async hookExport(request: HookExportRequest): Promise<DeepPartial<HookResponse>> {
-    const plugin = await this.#manager.pluginOrThrow('exporter-curl');
-    const data = await plugin.runExport(JSON.parse(request.request));
-    const info = { plugin: (await plugin.getInfo()).name };
-    return { info, data };
-  }
+const events = new EventChannel();
+const plugins: Record<string, PluginHandle> = {};

-  async hookImport(request: HookImportRequest): Promise<DeepPartial<HookResponse>> {
-    const plugins = await this.#manager.pluginsWith('import');
-    for (const p of plugins) {
-      const data = await p.runImport(request.data);
-      if (data != null && data !== 'null') {
-        const info = { plugin: (await p.getInfo()).name };
-        return { info, data };
-      }
-    }
-
-    throw new ServerError(Status.UNKNOWN, 'No importers found for data');
-  }
-
-  async hookResponseFilter(request: HookResponseFilterRequest): Promise<DeepPartial<HookResponse>> {
-    const pluginName = request.contentType.includes('json') ? 'filter-jsonpath' : 'filter-xpath';
-    const plugin = await this.#manager.pluginOrThrow(pluginName);
-    const data = await plugin.runResponseFilter(request);
-    const info = { plugin: (await plugin.getInfo()).name };
-    return { info, data };
-  }
-}
-
-let server = createServer();
-
-async function* errorHandlingMiddleware<Request, Response>(
-  call: ServerMiddlewareCall<Request, Response>,
-  context: CallContext,
-) {
+(async () => {
  try {
-    return yield* call.next(call.request, context);
-  } catch (error: unknown) {
-    if (error instanceof ServerError || isAbortError(error)) {
-      throw error;
+    for await (const e of client.eventStream(events.listen())) {
+      const pluginEvent: InternalEvent = JSON.parse(e.event);
+      // Handle special event to bootstrap plugin
+      if (pluginEvent.payload.type === 'boot_request') {
+        const plugin = new PluginHandle(pluginEvent.payload.dir, pluginEvent.pluginRefId, events);
+        plugins[pluginEvent.pluginRefId] = plugin;
+      }
+
+      // Once booted, forward all events to plugin's worker
+      const plugin = plugins[pluginEvent.pluginRefId];
+      if (!plugin) {
+        console.warn('Failed to get plugin for event by', pluginEvent.pluginRefId);
+        continue;
+      }
+
+      plugin.sendToWorker(pluginEvent);
    }
-
-    let details = String(error);
-
-    if (process.env.NODE_ENV === 'development') {
-      // @ts-ignore
-      details += `: ${error.stack}`;
+    console.log('Stream ended');
+  } catch (err: any) {
+    if (err.code === Status.CANCELLED) {
+      console.log('Stream was cancelled by server');
+    } else {
+      console.log('Client stream errored', err);
    }
-
-    throw new ServerError(Status.UNKNOWN, details);
  }
-}
-
-server = server.use(errorHandlingMiddleware);
-server.add(PluginRuntimeDefinition, new PluginRuntimeService());
-
-// Start on random port if YAAK_GRPC_PORT_FILE_PATH is set, or :4000
-const addr = process.env.YAAK_GRPC_PORT_FILE_PATH ? 'localhost:0' : 'localhost:4000';
-server.listen(addr).then((port) => {
-  console.log('gRPC server listening on', `http://localhost:${port}`);
-  if (process.env.YAAK_GRPC_PORT_FILE_PATH) {
-    console.log('Wrote port file to', process.env.YAAK_GRPC_PORT_FILE_PATH);
-    fs.writeFileSync(process.env.YAAK_GRPC_PORT_FILE_PATH, JSON.stringify({ port }, null, 2));
-  }
-});
+})();
--- a/plugin-runtime/src/index.worker.ts
+++ b/plugin-runtime/src/index.worker.ts
@@ -1,11 +1,13 @@
+import { ImportResponse, InternalEvent, InternalEventPayload } from '@yaakapp/api';
+import interceptStdout from 'intercept-stdout';
+import * as console from 'node:console';
 import { readFileSync } from 'node:fs';
 import path from 'node:path';
+import * as util from 'node:util';
 import { parentPort, workerData } from 'node:worker_threads';
-import { ParentToWorkerEvent } from './PluginHandle';
-import { PluginInfo } from './plugins';

 new Promise<void>(async (resolve, reject) => {
-  const { pluginDir } = workerData;
+  const { pluginDir /*, pluginRefId*/ } = workerData;
  const pathMod = path.join(pluginDir, 'build/index.js');
  const pathPkg = path.join(pluginDir, 'package.json');

@@ -18,59 +20,112 @@ new Promise<void>(async (resolve, reject) => {
    return;
  }

-  const mod = (await import(`file://${pathMod}`)).default ?? {};
+  prefixStdout(`[plugin][${pkg.name}] %s`);

-  const info: PluginInfo = {
-    capabilities: [],
-    name: pkg['name'] ?? 'n/a',
-    dir: pluginDir,
-  };
+  const mod = (await import(pathMod)).default ?? {};

-  if (typeof mod['pluginHookImport'] === 'function') {
-    info.capabilities.push('import');
-  }
+  const capabilities: string[] = [];
+  if (typeof mod.pluginHookExport === 'function') capabilities.push('export');
+  if (typeof mod.pluginHookImport === 'function') capabilities.push('import');
+  if (typeof mod.pluginHookResponseFilter === 'function') capabilities.push('filter');

-  if (typeof mod['pluginHookExport'] === 'function') {
-    info.capabilities.push('export');
-  }
+  console.log('Plugin initialized', pkg.name, capabilities, Object.keys(mod));

-  if (typeof mod['pluginHookResponseFilter'] === 'function') {
-    info.capabilities.push('filter');
-  }
+  // Message comes into the plugin to be processed
+  parentPort!.on('message', async ({ payload, pluginRefId, id: replyId }: InternalEvent) => {
+    console.log(`Received ${payload.type}`);

-  console.log('Loaded plugin', info.name, info.capabilities, info.dir);
-
-  function reply<T>(originalMsg: ParentToWorkerEvent, payload: T) {
-    parentPort!.postMessage({ payload, callbackId: originalMsg.callbackId });
-  }
-
-  function replyErr(originalMsg: ParentToWorkerEvent, error: unknown) {
-    parentPort!.postMessage({
-      error: String(error),
-      callbackId: originalMsg.callbackId,
-    });
-  }
-
-  parentPort!.on('message', async (msg: ParentToWorkerEvent) => {
    try {
-      const ctx = { todo: 'implement me' };
-      if (msg.name === 'run-import') {
-        reply(msg, await mod.pluginHookImport(ctx, msg.payload));
-      } else if (msg.name === 'run-filter') {
-        reply(msg, await mod.pluginHookResponseFilter(ctx, msg.payload));
-      } else if (msg.name === 'run-export') {
-        reply(msg, await mod.pluginHookExport(ctx, msg.payload));
-      } else if (msg.name === 'info') {
-        reply(msg, info);
-      } else {
-        console.log('Unknown message', msg);
+      if (payload.type === 'boot_request') {
+        const payload: InternalEventPayload = {
+          type: 'boot_response',
+          name: pkg.name,
+          version: pkg.version,
+          capabilities,
+        };
+        sendToServer({ id: genId(), pluginRefId, replyId, payload });
+        return;
      }
-    } catch (err: unknown) {
-      replyErr(msg, err);
+
+      if (payload.type === 'import_request' && typeof mod.pluginHookImport === 'function') {
+        const reply: ImportResponse | null = await mod.pluginHookImport({}, payload.content);
+        if (reply != null) {
+          const replyPayload: InternalEventPayload = {
+            type: 'import_response',
+            resources: reply?.resources,
+          };
+          sendToServer({ id: genId(), pluginRefId, replyId, payload: replyPayload });
+          return;
+        } else {
+          // Continue, to send back an empty reply
+        }
+      }
+
+      if (
+        payload.type === 'export_http_request_request' &&
+        typeof mod.pluginHookExport === 'function'
+      ) {
+        const reply: string = await mod.pluginHookExport({}, payload.httpRequest);
+        const replyPayload: InternalEventPayload = {
+          type: 'export_http_request_response',
+          content: reply,
+        };
+        sendToServer({ id: genId(), pluginRefId, replyId, payload: replyPayload });
+        return;
+      }
+
+      if (payload.type === 'filter_request' && typeof mod.pluginHookResponseFilter === 'function') {
+        const reply: string = await mod.pluginHookResponseFilter(
+          {},
+          { filter: payload.filter, body: payload.content },
+        );
+        const replyPayload: InternalEventPayload = {
+          type: 'filter_response',
+          items: JSON.parse(reply),
+        };
+        sendToServer({ id: genId(), pluginRefId, replyId, payload: replyPayload });
+        return;
+      }
+    } catch (err) {
+      console.log('Plugin call threw exception', payload.type, err);
+      // TODO: Return errors to server
    }
+
+    // No matches, so send back an empty response so the caller doesn't block forever
+    const id = genId();
+    console.log('Sending nothing back to', id, { replyId });
+    sendToServer({ id, pluginRefId, replyId, payload: { type: 'empty_response' } });
  });

  resolve();
 }).catch((err) => {
  console.log('failed to boot plugin', err);
 });
+
+function sendToServer(e: InternalEvent) {
+  parentPort!.postMessage(e);
+}
+
+function genId(len = 5): string {
+  const alphabet = '01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
+  let id = '';
+  for (let i = 0; i < len; i++) {
+    id += alphabet[Math.floor(Math.random() * alphabet.length)];
+  }
+  return id;
+}
+
+function prefixStdout(s: string) {
+  if (!s.includes('%s')) {
+    throw new Error('Console prefix must contain a "%s" replacer');
+  }
+  interceptStdout((text) => {
+    const lines = text.split(/\n/);
+    let newText = '';
+    for (let i = 0; i < lines.length; i++) {
+      if (lines[i] == '') continue;
+      newText += util.format(s, lines[i]) + '\n';
+    }
+    return newText.trimEnd();
+  });
+}
--- a/plugin-runtime/src/plugins.ts
+++ b/plugin-runtime/src/plugins.ts
@@ -1,18 +0,0 @@
-import * as fs from 'node:fs';
-import path from 'node:path';
-import { PluginHandle } from './PluginHandle';
-
-export interface PluginInfo {
-  name: string;
-  dir: string;
-  capabilities: ('import' | 'export' | 'filter')[];
-}
-
-export function loadPlugins(): PluginHandle[] {
-  const pluginsDir = process.env.YAAK_PLUGINS_DIR;
-  if (!pluginsDir) throw new Error('YAAK_PLUGINS_DIR is not set');
-  console.log('Loading plugins from', pluginsDir);
-
-  const pluginDirs = fs.readdirSync(pluginsDir).map((p) => path.join(pluginsDir, p));
-  return pluginDirs.map((pluginDir) => new PluginHandle(pluginDir));
-}