🚚 split logic in two seperate files, create nio_llm module

This commit is contained in:
Laureηt 2023-05-24 21:57:48 +02:00
parent aa6e8e79c2
commit 69e24bfbd6
Signed by: Laurent
SSH key fingerprint: SHA256:kZEpW8cMJ54PDeCvOhzreNr4FSh6R13CMGH/POoO8DI
4 changed files with 104 additions and 119 deletions

View file

@ -1,28 +0,0 @@
from textwrap import dedent
from llama_cpp import Llama
llm = Llama(model_path="../../../llama.cpp/models/sv13B/stable-vicuna-13B.ggml.q5_1.bin", n_threads=12)
msg = dedent(
"""
You are pipobot, an arrogant assistant. Answer as concisely as possible.
<@fainsil:inpt.fr>: Qu'est ce qu'une intégrale de Lebesgue ?
<@pipobot:inpt.fr>:
""",
).strip()
print(msg)
print(repr(msg))
output = llm(
msg,
max_tokens=100,
stop=["<@fainsil:inpt.fr>:", "\n"],
echo=True,
)
print(output)
res = output["choices"][0]["text"]
print(res)
print(res.removeprefix(msg).strip())

0
nio_llm/__init__.py Normal file
View file

View file

@ -1,17 +1,14 @@
"""A Matrix client that uses Llama to respond to messages.""" """A Matrix client that uses Llama to respond to messages."""
import asyncio
import logging import logging
import time import time
from pathlib import Path
from textwrap import dedent from textwrap import dedent
import click
from huggingface_hub import hf_hub_download
from llama_cpp import Llama from llama_cpp import Llama
from nio import AsyncClient, MatrixRoom, RoomMessageText from nio import AsyncClient, MatrixRoom, RoomMessageText
from rich.logging import RichHandler
logger = logging.getLogger("nio-llm") logger = logging.getLogger("nio-llm.client")
class LLMClient(AsyncClient): class LLMClient(AsyncClient):
@ -23,7 +20,7 @@ class LLMClient(AsyncClient):
homeserver: str, homeserver: str,
device_id: str, device_id: str,
preprompt: str, preprompt: str,
ggml_path: str, ggml_path: Path,
room: str, room: str,
): ):
"""Create a new LLMClient instance.""" """Create a new LLMClient instance."""
@ -42,7 +39,7 @@ class LLMClient(AsyncClient):
# create the Llama instance # create the Llama instance
self.llm = Llama( self.llm = Llama(
model_path=ggml_path, model_path=str(ggml_path),
n_threads=12, n_threads=12,
) )
@ -125,87 +122,3 @@ class LLMClient(AsyncClient):
"body": output, "body": output,
}, },
) )
@click.command()
@click.option("--homeserver", "-h", help="The homeserver to connect to.", required=True)
@click.option("--device-id", "-d", help="The device ID to use.", required=True)
@click.option("--username", "-u", help="The username to log in as.", required=True)
@click.option("--password", "-p", help="The password to log in with.", required=True)
@click.option("--room", "-r", help="The room to join.", required=True)
@click.option("--preprompt", "-t", help="The preprompt to use.", required=True)
def main(
homeserver: str,
device_id: str,
username: str,
password: str,
room: str,
preprompt: str,
) -> None:
"""Run the main program.
Download the model from HuggingFace Hub and start the async loop.
"""
# download the model
ggml_path = hf_hub_download(
repo_id="TheBloke/stable-vicuna-13B-GGML",
filename="stable-vicuna-13B.ggmlv3.q5_1.bin",
)
asyncio.get_event_loop().run_until_complete(
_main(
ggml_path=ggml_path,
homeserver=homeserver,
device_id=device_id,
username=username,
password=password,
preprompt=preprompt,
room=room,
),
)
async def _main(
homeserver: str,
device_id: str,
username: str,
password: str,
room: str,
preprompt: str,
ggml_path: str,
) -> None:
"""Run the async main program.
Create the client, login, join the room, and sync forever.
"""
# create the client
client = LLMClient(
homeserver=homeserver,
device_id=device_id,
username=username,
room=room,
preprompt=preprompt,
ggml_path=ggml_path,
)
# Login to the homeserver
logger.debug(await client.login(password))
# Join the room, if not already joined
logger.debug(await client.join(room))
# Sync with the server forever
await client.sync_forever(timeout=30000)
if __name__ == "__main__":
# set up logging
logging.captureWarnings(True)
logging.basicConfig(
level="DEBUG",
format="%(name)s: %(message)s",
handlers=[RichHandler(markup=True)],
)
# run the main program (with environment variables)
main(auto_envvar_prefix="NIOLLM")

100
nio_llm/main.py Normal file
View file

@ -0,0 +1,100 @@
"""The main program for nio-llm."""
import asyncio
import logging
from pathlib import Path
import click
from huggingface_hub import hf_hub_download
from rich.logging import RichHandler
from nio_llm.client import LLMClient
logger = logging.getLogger("nio-llm.main")
@click.command()
@click.option("--homeserver", "-h", help="The homeserver to connect to.", required=True)
@click.option("--device-id", "-d", help="The device ID to use.", required=True)
@click.option("--username", "-u", help="The username to log in as.", required=True)
@click.option("--password", "-p", help="The password to log in with.", required=True)
@click.option("--room", "-r", help="The room to join.", required=True)
@click.option("--preprompt", "-t", help="The preprompt to use.", required=True)
def main(
homeserver: str,
device_id: str,
username: str,
password: str,
room: str,
preprompt: str,
) -> None:
"""Run the main program.
Download the model from HuggingFace Hub and start the async loop.
"""
# download the model
ggml_path = Path(
hf_hub_download(
repo_id="TheBloke/stable-vicuna-13B-GGML",
filename="stable-vicuna-13B.ggmlv3.q5_1.bin",
),
)
# start the async loop
asyncio.get_event_loop().run_until_complete(
_main(
ggml_path=ggml_path,
homeserver=homeserver,
device_id=device_id,
username=username,
password=password,
preprompt=preprompt,
room=room,
),
)
async def _main(
homeserver: str,
device_id: str,
username: str,
password: str,
room: str,
preprompt: str,
ggml_path: Path,
) -> None:
"""Run the async main program.
Create the client, login, join the room, and sync forever.
"""
# create the client
client = LLMClient(
homeserver=homeserver,
device_id=device_id,
username=username,
room=room,
preprompt=preprompt,
ggml_path=ggml_path,
)
# Login to the homeserver
logger.debug(await client.login(password))
# Join the room, if not already joined
logger.debug(await client.join(room))
# Sync with the server forever
await client.sync_forever(timeout=30000)
if __name__ == "__main__":
# set up logging
logging.captureWarnings(True)
logging.basicConfig(
level="DEBUG",
format="%(name)s: %(message)s",
handlers=[RichHandler(markup=True)],
)
# run the main program (with environment variables)
main(auto_envvar_prefix="NIOLLM")