LLM integration proof-of-concept

js/places/placesService.js

@@ -137,6 +137,8 @@ function handleRequest (data, cb) {
   }
 
   if (action === 'updatePlace') {
+    console.time()
+    llmAdapter.summarizePage(pageData.extractedText).then(res => {console.timeEnd(); console.log("got summary", res)})
     db.transaction('rw', db.places, function () {
       db.places.where('url').equals(pageData.url).first(function (item) {
         var isNewItem = false
@@ -292,3 +294,54 @@ ipcRenderer.on('places-connect', function (e) {
   })
   e.ports[0].start()
 })
+
+// Initialize LLM service
+
+const llmAdapter = {
+  messagePort: null,
+  pendingPromises: {},
+  invokeWithPromise: function (data) {
+    const callbackId = Math.random()
+    const { promise, resolve, reject } = Promise.withResolvers()
+    llmAdapter.pendingPromises[callbackId] = { promise, resolve, reject }
+    llmAdapter.messagePort.postMessage({
+      ...data,
+      callbackId
+    })
+    return promise
+  },
+  replyToPromise: function (callbackId, result) {
+    if (llmAdapter.pendingPromises[callbackId]) {
+      llmAdapter.pendingPromises[callbackId].resolve(result)
+      delete llmAdapter.pendingPromises[callbackId]
+    } else {
+      throw new Error('missing callbackId')
+    }
+  },
+  summarizePage: function (text) {
+    const textToSummarize = text.substring(0, 20000)
+    console.log("getting summary", textToSummarize)
+    return llmAdapter.invokeWithPromise({
+      action: 'run',
+      // TODO there's some character encoding problem when passing text to the subprocess; encodeURIComponent seems to help
+      input: encodeURIComponent(`Write one short sentence summarizing the key point of the text. Be concise. Do not mention the author. Use the following template for your response: [The key point is that]: [your answer]. \n` + textToSummarize)
+    })
+  },
+  onMessage: function (e) {
+    if (e.data.callbackId) {
+      llmAdapter.replyToPromise(e.data.callbackId, e.data.result)
+    }
+  },
+  initialize: () => {
+    const { port1, port2 } = new MessageChannel()
+
+    ipcRenderer.postMessage('llm-service-connect', null, [port1])
+
+    llmAdapter.messagePort = port2
+    port2.addEventListener('message', llmAdapter.onMessage)
+
+    port2.start()
+  }
+}
+
+llmAdapter.initialize()

main/llmService.mjs

@@ -0,0 +1,50 @@
+import { getLlama, LlamaChatSession } from 'node-llama-cpp'
+
+process.parentPort.on('message', (e) => {
+  e.ports[0].addListener('message', function (e2) {
+    console.log(e2)
+    handleRequest(e2.data).then(res => e.ports[0].postMessage(res))
+  })
+  e.ports[0].start()
+})
+
+let model
+let context
+let session
+
+async function ensureModelLoaded () {
+  if (!model) {
+    console.log('run')
+    const llama = await getLlama()
+    console.log('loaded')
+    model = await llama.loadModel({
+      // TODO update
+      // modelPath: 'Llama-3.2-1B-Instruct-Q6_K_L.gguf'
+      // modelPath: 'Llama-3.2-3B-Instruct-Q4_K_M.gguf'
+    })
+    context = await model.createContext()
+    session = new LlamaChatSession({
+      contextSequence: context.getSequence()
+    })
+  }
+}
+
+async function handleRequest (data) {
+  console.log(data)
+
+  await ensureModelLoaded()
+
+  if (data.action === 'run') {
+    console.log("input: " + decodeURIComponent(data.input))
+    const result = await session.prompt(decodeURIComponent(data.input))
+
+    console.log('result: ' + result)
+
+    session.resetChatHistory()
+
+    return {
+      result,
+      callbackId: data.callbackId
+    }
+  }
+}

main/main.js

@@ -16,7 +16,9 @@ const {
   nativeTheme,
   shell,
   net,
-  WebContentsView
+  WebContentsView,
+  utilityProcess,
+  MessageChannelMain
 } = electron
 
 crashReporter.start({
@@ -494,4 +496,13 @@ ipc.on('places-connect', function (e) {
 
 function getWindowWebContents (win) {
   return win.getContentView().children[0].webContents
-}
+}
+
+app.on('ready', () => {
+  // Main process
+  const llmServiceProcess = utilityProcess.fork(path.join(__dirname, "main/llmService.mjs"))
+
+  ipc.on('llm-service-connect', function(e) {
+    llmServiceProcess.postMessage({message: 'init'}, e.ports)
+  })
+})

package.json

@@ -36,6 +36,7 @@
     "electron-squirrel-startup": "^1.0.0",
     "expr-eval": "^2.0.2",
     "node-abi": "^3.8.0",
+    "node-llama-cpp": "^3.3.2",
     "pdfjs-dist": "4.2.67",
     "quick-score": "^0.2.0",
     "regedit": "^3.0.3",