2023-09-01 14:50:41 +00:00
|
|
|
from refiners.fluxion.adapters.lora import Lora, SingleLoraAdapter, LoraAdapter
|
2023-08-04 13:28:41 +00:00
|
|
|
from torch import randn, allclose
|
|
|
|
|
import refiners.fluxion.layers as fl
|
|
|
|
|
|
|
|
|
|
|
2023-08-31 08:40:01 +00:00
|
|
|
def test_single_lora_adapter() -> None:
|
2023-08-04 13:28:41 +00:00
|
|
|
chain = fl.Chain(
|
|
|
|
|
fl.Chain(
|
|
|
|
|
fl.Linear(in_features=1, out_features=1),
|
|
|
|
|
fl.Linear(in_features=1, out_features=1),
|
|
|
|
|
),
|
|
|
|
|
fl.Linear(in_features=1, out_features=2),
|
|
|
|
|
)
|
|
|
|
|
x = randn(1, 1)
|
|
|
|
|
y = chain(x)
|
|
|
|
|
|
2023-08-31 08:40:01 +00:00
|
|
|
lora_adapter = SingleLoraAdapter(chain.Chain.Linear_1).inject(chain.Chain)
|
2023-08-04 13:28:41 +00:00
|
|
|
|
|
|
|
|
assert isinstance(lora_adapter[1], Lora)
|
|
|
|
|
assert allclose(input=chain(x), other=y)
|
|
|
|
|
assert lora_adapter.parent == chain.Chain
|
|
|
|
|
|
|
|
|
|
lora_adapter.eject()
|
|
|
|
|
assert isinstance(chain.Chain[0], fl.Linear)
|
|
|
|
|
assert len(chain) == 2
|
|
|
|
|
|
|
|
|
|
lora_adapter.inject(chain.Chain)
|
2023-08-31 08:40:01 +00:00
|
|
|
assert isinstance(chain.Chain[0], SingleLoraAdapter)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_lora_adapter() -> None:
|
|
|
|
|
chain = fl.Chain(
|
|
|
|
|
fl.Chain(
|
|
|
|
|
fl.Linear(in_features=1, out_features=1),
|
|
|
|
|
fl.Linear(in_features=1, out_features=1),
|
|
|
|
|
),
|
|
|
|
|
fl.Linear(in_features=1, out_features=2),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
LoraAdapter[fl.Chain](chain, sub_targets=chain.walk(fl.Linear), rank=1, scale=1.0).inject()
|
|
|
|
|
|
|
|
|
|
assert len(list(chain.layers(Lora))) == 3
|
