llama.cpp/common/ngram-mod.h at 820ad0d191c39cc48cc165d6568082ed8d974788 · ggml-org/llama.cpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
#pragma once

#include <cstdint>
#include <vector>
#include <cstddef>

//
// common_ngram_mod
// ref: https://github.com/ggml-org/llama.cpp/pull/19164
//

// basic n-gram hasher
struct common_ngram_mod {
    using entry_t = int32_t;

    static constexpr entry_t EMPTY = -1;

    static constexpr int8_t SCORE_INIT = 0;
    static constexpr int8_t SCORE_MIN  = -5;
    static constexpr int8_t SCORE_MAX  = 20;
    static constexpr int8_t SCORE_THR  = 0;
    static constexpr int8_t SCORE_INS  = 3;

    common_ngram_mod(uint16_t n, size_t size);

    size_t  idx(const entry_t * tokens) const;
    void    add(const entry_t * tokens);
    entry_t get(const entry_t * tokens) const; // return -1 if not found

    void reset();

    // expose the hash index for external bookkeeping
    size_t index(const entry_t * tokens) const;

    // score handling
    void inc_score(const entry_t * tokens);
    void dec_score(const entry_t * tokens);
    void inc_score_by_index(size_t i);
    void dec_score_by_index(size_t i);
    void prune_low_score(); // remove entries below SCORE_THR

    size_t get_n()    const;
    size_t get_used() const;

    void update_score_stats();

    size_t get_collisions() const;
    size_t get_below_thr()  const;
    size_t get_at_min()     const;
    size_t get_at_max()     const;
    size_t get_at_ins()    const;

    size_t size()       const;
    size_t size_bytes() const;

private:
    size_t n; // ngram size to hash

    size_t used;

    std::vector<entry_t> entries;
    // per-entry score, range SCORE_MIN .. SCORE_MAX
    std::vector<int8_t> scores;

    // stats
    // count of hash collisions
    size_t collisions = 0;
    // counts for score
    size_t count_below_thr = 0;
    size_t count_at_min   = 0;
    size_t count_at_max   = 0;
    size_t count_at_ins  = 0;
};