Laurent/nio-llm
1
0
Fork 0
mirror of https://github.com/Laurent2916/nio-llm.git synced 2024-11-21 21:58:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

⚡️ use async openai methods + create async task for the typing loop

Browse source
This commit is contained in:
Laureηt 2023-10-19 16:10:47 +00:00
parent 8eda4825d9
commit 0f312a0a70
Signed by: Laurent
SSH key fingerprint: SHA256:kZEpW8cMJ54PDeCvOhzreNr4FSh6R13CMGH/POoO8DI
1 changed files with 34 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
src/nio_llm/client.py
View file

@ -1,3 +1,4 @@
import asyncio
import logging import logging
import time import time
from collections import deque from collections import deque
@ -45,7 +46,7 @@ class LLMClient(AsyncClient):
The OpenAI temperature to use. The OpenAI temperature to use.
openai_max_tokens (`int`): openai_max_tokens (`int`):
The OpenAI max tokens to use. The OpenAI max tokens to use.
history_size (`int`, default `3`): history_size (`int`):
The number of messages to keep in history. The number of messages to keep in history.
""" """
self.uid = f"@{username}:{homeserver.removeprefix('https://')}" self.uid = f"@{username}:{homeserver.removeprefix('https://')}"
@ -73,7 +74,23 @@ class LLMClient(AsyncClient):
# add callbacks # add callbacks
self.add_event_callback(self.message_callback, RoomMessageText) # type: ignore self.add_event_callback(self.message_callback, RoomMessageText) # type: ignore
async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> None: async def typing_loop(self) -> None:
"""Send typing indicators every 5 seconds."""
logging.debug("Started typing indicator.")
try:
while True:
logging.debug("Sending typing indicator.")
await self.room_typing(self.room, True)
await asyncio.sleep(5)
except asyncio.CancelledError:
await self.room_typing(self.room, False)
logging.debug("Stopped typing indicator.")
async def message_callback(
self,
room: MatrixRoom,
event: RoomMessageText,
) -> None:
"""Process new messages as they come in. """Process new messages as they come in.
Args: Args:
@ -128,16 +145,11 @@ class LLMClient(AsyncClient):
logger.debug("Ignoring message not mentioning us.") logger.debug("Ignoring message not mentioning us.")
return return
# enable typing indicator # start typing indicator loop
await self.room_typing( typing_task = asyncio.create_task(self.typing_loop())
self.room,
typing_state=True,
timeout=30000,
)
logger.debug("Enabled typing indicator.")
# generate response using llama.cpp # generate response using llama.cpp
response = openai.ChatCompletion.create( response = await openai.ChatCompletion.acreate(
model="local-model", model="local-model",
messages=[ messages=[
{ {
@ -162,10 +174,6 @@ class LLMClient(AsyncClient):
output = response["choices"][0]["message"]["content"] # type: ignore output = response["choices"][0]["message"]["content"] # type: ignore
output = output.strip().removeprefix(f"{self.uid}:").strip() output = output.strip().removeprefix(f"{self.uid}:").strip()
# disable typing indicator
await self.room_typing(self.room, typing_state=False)
logger.debug("Disabled typing indicator.")
# send the response # send the response
await self.room_send( await self.room_send(
room_id=self.room, room_id=self.room,
@ -177,12 +185,21 @@ class LLMClient(AsyncClient):
) )
logger.debug(f"Sent response: {output}") logger.debug(f"Sent response: {output}")
async def start(self, password, sync_timeout=30000) -> None: # stop typing indicator loop
typing_task.cancel()
async def start(
self,
password: str,
sync_timeout: int = 30000,
) -> None:
"""Start the client. """Start the client.
Args: Args:
password (`str`): The password to log in with. password (`str`):
sync_timeout (`int`, default `30000`): The sync timeout in milliseconds. The password to log in with.
sync_timeout (`int`, default `30000`):
The sync timeout in milliseconds.
""" """
# Login to the homeserver # Login to the homeserver
logger.debug(await self.login(password)) logger.debug(await self.login(password))