mirror of
https://github.com/Laurent2916/nio-llm.git
synced 2024-11-09 23:12:05 +00:00
⚡️ use async openai methods + create async task for the typing loop
This commit is contained in:
parent
8eda4825d9
commit
0f312a0a70
|
@ -1,3 +1,4 @@
|
||||||
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from collections import deque
|
from collections import deque
|
||||||
|
@ -45,7 +46,7 @@ class LLMClient(AsyncClient):
|
||||||
The OpenAI temperature to use.
|
The OpenAI temperature to use.
|
||||||
openai_max_tokens (`int`):
|
openai_max_tokens (`int`):
|
||||||
The OpenAI max tokens to use.
|
The OpenAI max tokens to use.
|
||||||
history_size (`int`, default `3`):
|
history_size (`int`):
|
||||||
The number of messages to keep in history.
|
The number of messages to keep in history.
|
||||||
"""
|
"""
|
||||||
self.uid = f"@{username}:{homeserver.removeprefix('https://')}"
|
self.uid = f"@{username}:{homeserver.removeprefix('https://')}"
|
||||||
|
@ -73,7 +74,23 @@ class LLMClient(AsyncClient):
|
||||||
# add callbacks
|
# add callbacks
|
||||||
self.add_event_callback(self.message_callback, RoomMessageText) # type: ignore
|
self.add_event_callback(self.message_callback, RoomMessageText) # type: ignore
|
||||||
|
|
||||||
async def message_callback(self, room: MatrixRoom, event: RoomMessageText) -> None:
|
async def typing_loop(self) -> None:
|
||||||
|
"""Send typing indicators every 5 seconds."""
|
||||||
|
logging.debug("Started typing indicator.")
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
logging.debug("Sending typing indicator.")
|
||||||
|
await self.room_typing(self.room, True)
|
||||||
|
await asyncio.sleep(5)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
await self.room_typing(self.room, False)
|
||||||
|
logging.debug("Stopped typing indicator.")
|
||||||
|
|
||||||
|
async def message_callback(
|
||||||
|
self,
|
||||||
|
room: MatrixRoom,
|
||||||
|
event: RoomMessageText,
|
||||||
|
) -> None:
|
||||||
"""Process new messages as they come in.
|
"""Process new messages as they come in.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
@ -128,16 +145,11 @@ class LLMClient(AsyncClient):
|
||||||
logger.debug("Ignoring message not mentioning us.")
|
logger.debug("Ignoring message not mentioning us.")
|
||||||
return
|
return
|
||||||
|
|
||||||
# enable typing indicator
|
# start typing indicator loop
|
||||||
await self.room_typing(
|
typing_task = asyncio.create_task(self.typing_loop())
|
||||||
self.room,
|
|
||||||
typing_state=True,
|
|
||||||
timeout=30000,
|
|
||||||
)
|
|
||||||
logger.debug("Enabled typing indicator.")
|
|
||||||
|
|
||||||
# generate response using llama.cpp
|
# generate response using llama.cpp
|
||||||
response = openai.ChatCompletion.create(
|
response = await openai.ChatCompletion.acreate(
|
||||||
model="local-model",
|
model="local-model",
|
||||||
messages=[
|
messages=[
|
||||||
{
|
{
|
||||||
|
@ -162,10 +174,6 @@ class LLMClient(AsyncClient):
|
||||||
output = response["choices"][0]["message"]["content"] # type: ignore
|
output = response["choices"][0]["message"]["content"] # type: ignore
|
||||||
output = output.strip().removeprefix(f"{self.uid}:").strip()
|
output = output.strip().removeprefix(f"{self.uid}:").strip()
|
||||||
|
|
||||||
# disable typing indicator
|
|
||||||
await self.room_typing(self.room, typing_state=False)
|
|
||||||
logger.debug("Disabled typing indicator.")
|
|
||||||
|
|
||||||
# send the response
|
# send the response
|
||||||
await self.room_send(
|
await self.room_send(
|
||||||
room_id=self.room,
|
room_id=self.room,
|
||||||
|
@ -177,12 +185,21 @@ class LLMClient(AsyncClient):
|
||||||
)
|
)
|
||||||
logger.debug(f"Sent response: {output}")
|
logger.debug(f"Sent response: {output}")
|
||||||
|
|
||||||
async def start(self, password, sync_timeout=30000) -> None:
|
# stop typing indicator loop
|
||||||
|
typing_task.cancel()
|
||||||
|
|
||||||
|
async def start(
|
||||||
|
self,
|
||||||
|
password: str,
|
||||||
|
sync_timeout: int = 30000,
|
||||||
|
) -> None:
|
||||||
"""Start the client.
|
"""Start the client.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
password (`str`): The password to log in with.
|
password (`str`):
|
||||||
sync_timeout (`int`, default `30000`): The sync timeout in milliseconds.
|
The password to log in with.
|
||||||
|
sync_timeout (`int`, default `30000`):
|
||||||
|
The sync timeout in milliseconds.
|
||||||
"""
|
"""
|
||||||
# Login to the homeserver
|
# Login to the homeserver
|
||||||
logger.debug(await self.login(password))
|
logger.debug(await self.login(password))
|
||||||
|
|
Loading…
Reference in a new issue