Skip to content

Fix rope scaling defaults #767

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 29, 2023
Merged

Fix rope scaling defaults #767

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
813da01
Fix rope scale with backwards compatibility
Josh-XT Sep 29, 2023
9eaffdb
Fix defaults
Josh-XT Sep 29, 2023
9997432
Fix op
Josh-XT Sep 29, 2023
0bd22a4
Remove backwards compatibility
Josh-XT Sep 29, 2023
92144a6
Check single val
Josh-XT Sep 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions llama_cpp/llama.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,8 +229,8 @@ def __init__(
n_batch: int = 512,
n_threads: Optional[int] = None,
n_threads_batch: Optional[int] = None,
rope_freq_base: float = 10000.0,
rope_freq_scale: float = 1.0,
rope_freq_base: float = 0.0,
rope_freq_scale: float = 0.0,
mul_mat_q: bool = True,
f16_kv: bool = True,
logits_all: bool = False,
Expand Down Expand Up @@ -282,7 +282,6 @@ def __init__(
Returns:
A Llama instance.
"""

self.verbose = verbose

self.numa = numa
Expand Down Expand Up @@ -320,16 +319,19 @@ def __init__(
self.n_threads_batch = n_threads_batch or max(
multiprocessing.cpu_count() // 2, 1
)

# Context Params
self.context_params = llama_cpp.llama_context_default_params()
self.context_params.seed = seed
self.context_params.n_ctx = n_ctx
self.context_params.n_batch = self.n_batch
self.context_params.n_threads = self.n_threads
self.context_params.n_threads_batch = self.n_threads_batch
self.context_params.rope_freq_base = rope_freq_base
self.context_params.rope_freq_scale = rope_freq_scale
self.context_params.rope_freq_base = (
rope_freq_base if rope_freq_base != 0.0 else 0
)
self.context_params.rope_freq_scale = (
rope_freq_scale if rope_freq_scale != 0.0 else 0
)
self.context_params.mul_mat_q = mul_mat_q
self.context_params.f16_kv = f16_kv
self.context_params.logits_all = logits_all
Expand All @@ -338,7 +340,6 @@ def __init__(
# Sampling Params
self.last_n_tokens_size = last_n_tokens_size


self.cache: Optional[BaseLlamaCache] = None

self.lora_base = lora_base
Expand Down