diff --git a/js/places/placesService.js b/js/places/placesService.js index 47e8d1342..b1e92654a 100644 --- a/js/places/placesService.js +++ b/js/places/placesService.js @@ -137,6 +137,8 @@ function handleRequest (data, cb) { } if (action === 'updatePlace') { + console.time() + llmAdapter.summarizePage(pageData.extractedText).then(res => {console.timeEnd(); console.log("got summary", res)}) db.transaction('rw', db.places, function () { db.places.where('url').equals(pageData.url).first(function (item) { var isNewItem = false @@ -292,3 +294,54 @@ ipcRenderer.on('places-connect', function (e) { }) e.ports[0].start() }) + +// Initialize LLM service + +const llmAdapter = { + messagePort: null, + pendingPromises: {}, + invokeWithPromise: function (data) { + const callbackId = Math.random() + const { promise, resolve, reject } = Promise.withResolvers() + llmAdapter.pendingPromises[callbackId] = { promise, resolve, reject } + llmAdapter.messagePort.postMessage({ + ...data, + callbackId + }) + return promise + }, + replyToPromise: function (callbackId, result) { + if (llmAdapter.pendingPromises[callbackId]) { + llmAdapter.pendingPromises[callbackId].resolve(result) + delete llmAdapter.pendingPromises[callbackId] + } else { + throw new Error('missing callbackId') + } + }, + summarizePage: function (text) { + const textToSummarize = text.substring(0, 20000) + console.log("getting summary", textToSummarize) + return llmAdapter.invokeWithPromise({ + action: 'run', + // TODO there's some character encoding problem when passing text to the subprocess; encodeURIComponent seems to help + input: encodeURIComponent(`Write one short sentence summarizing the key point of the text. Be concise. Do not mention the author. Use the following template for your response: [The key point is that]: [your answer]. \n` + textToSummarize) + }) + }, + onMessage: function (e) { + if (e.data.callbackId) { + llmAdapter.replyToPromise(e.data.callbackId, e.data.result) + } + }, + initialize: () => { + const { port1, port2 } = new MessageChannel() + + ipcRenderer.postMessage('llm-service-connect', null, [port1]) + + llmAdapter.messagePort = port2 + port2.addEventListener('message', llmAdapter.onMessage) + + port2.start() + } +} + +llmAdapter.initialize() diff --git a/main/llmService.mjs b/main/llmService.mjs new file mode 100644 index 000000000..563dca530 --- /dev/null +++ b/main/llmService.mjs @@ -0,0 +1,50 @@ +import { getLlama, LlamaChatSession } from 'node-llama-cpp' + +process.parentPort.on('message', (e) => { + e.ports[0].addListener('message', function (e2) { + console.log(e2) + handleRequest(e2.data).then(res => e.ports[0].postMessage(res)) + }) + e.ports[0].start() +}) + +let model +let context +let session + +async function ensureModelLoaded () { + if (!model) { + console.log('run') + const llama = await getLlama() + console.log('loaded') + model = await llama.loadModel({ + // TODO update + // modelPath: 'Llama-3.2-1B-Instruct-Q6_K_L.gguf' + // modelPath: 'Llama-3.2-3B-Instruct-Q4_K_M.gguf' + }) + context = await model.createContext() + session = new LlamaChatSession({ + contextSequence: context.getSequence() + }) + } +} + +async function handleRequest (data) { + console.log(data) + + await ensureModelLoaded() + + if (data.action === 'run') { + console.log("input: " + decodeURIComponent(data.input)) + const result = await session.prompt(decodeURIComponent(data.input)) + + console.log('result: ' + result) + + session.resetChatHistory() + + return { + result, + callbackId: data.callbackId + } + } +} diff --git a/main/main.js b/main/main.js index a0832f007..57fffb4b1 100644 --- a/main/main.js +++ b/main/main.js @@ -16,7 +16,9 @@ const { nativeTheme, shell, net, - WebContentsView + WebContentsView, + utilityProcess, + MessageChannelMain } = electron crashReporter.start({ @@ -494,4 +496,13 @@ ipc.on('places-connect', function (e) { function getWindowWebContents (win) { return win.getContentView().children[0].webContents -} \ No newline at end of file +} + +app.on('ready', () => { + // Main process + const llmServiceProcess = utilityProcess.fork(path.join(__dirname, "main/llmService.mjs")) + + ipc.on('llm-service-connect', function(e) { + llmServiceProcess.postMessage({message: 'init'}, e.ports) + }) +}) diff --git a/package.json b/package.json index 0df9e95e8..c0baaed05 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "electron-squirrel-startup": "^1.0.0", "expr-eval": "^2.0.2", "node-abi": "^3.8.0", + "node-llama-cpp": "^3.3.2", "pdfjs-dist": "4.2.67", "quick-score": "^0.2.0", "regedit": "^3.0.3",