RWKV (pronounced RwaKuv) is an RNN with great LLM performance, which can also be directly trained like a GPT transformer (parallelizable). We are at RWKV-7 "Goose". So it's combining the best of RNN and transformer - great performance, linear time, constant space (no kv-cache), fast training, infinite ctx_len, and free sentence embedding.
271 matches across 9 categories. Click a row to expand file-level details.
| Severity | File | Line | Snippet |
|---|---|---|---|
| LOW | RWKV-v7/rwkv_v7_demo.py | 5 | |
| LOW | RWKV-v7/rwkv_v7_demo.py | 5 | |
| LOW | RWKV-v7/rwkv_mmlu_eval.py | 4 | |
| LOW | RWKV-v7/rwkv_mmlu_eval.py | 4 | |
| LOW | RWKV-v7/rwkv_mmlu_eval.py | 14 | |
| LOW | RWKV-v7/train_temp/rwkv7_train_simplified.py | 5 | |
| LOW | RWKV-v7/train_temp/train.py | 11 | |
| LOW | RWKV-v7/train_temp/train.py | 65 | |
| LOW | RWKV-v7/train_temp/train.py | 65 | |
| LOW | RWKV-v7/train_temp/train.py | 65 | |
| LOW | RWKV-v7/train_temp/src/model.py | 5 | |
| LOW | RWKV-v7/train_temp/src/model.py | 8 | |
| LOW | RWKV-v7/train_temp/src/model.py | 10 | |
| LOW | RWKV-v7/train_temp/src/model.py | 10 | |
| LOW | RWKV-v7/train_temp/src/dataset.py | 5 | |
| LOW | RWKV-v7/train_temp/src/dataset.py | 5 | |
| LOW | RWKV-v7/train_temp/src/dataset.py | 5 | |
| LOW | RWKV-v7/train_temp/src/dataset.py | 5 | |
| LOW | RWKV-v7/train_temp/src/dataset.py | 6 | |
| LOW | RWKV-v7/train_temp/src/trainer.py | 1 | |
| LOW | RWKV-v7/train_temp/src/trainer.py | 1 | |
| LOW | RWKV-v7/train_temp/src/trainer.py | 3 | |
| LOW | RWKV-v7/train_temp/src/trainer.py | 5 | |
| LOW | RWKV-v8/251016_rosa_1bit_run.py | 3 | |
| LOW | RWKV-v8/251018_rosa_4bit_run.py | 3 | |
| LOW | RWKV-v8/260222_rosa4bitLM_L12.py | 5 | |
| LOW | RWKV-v8/260222_rosa4bitLM_L12.py | 5 | |
| LOW | RWKV-v8/260222_rosa4bitLM_L12.py | 5 | |
| LOW | RWKV-v8/260222_rosa4bitLM_L12.py | 5 | |
| LOW | RWKV-v8/260212_rosa1bitLM_L12.py | 5 | |
| LOW | RWKV-v8/260212_rosa1bitLM_L12.py | 5 | |
| LOW | RWKV-v8/260212_rosa1bitLM_L12.py | 5 | |
| LOW | RWKV-v8/260212_rosa1bitLM_L12.py | 5 | |
| LOW | RWKV-v1/train.py | 5 | |
| LOW | RWKV-v1/train.py | 5 | |
| LOW | RWKV-v1/train.py | 5 | |
| LOW | RWKV-v1/train.py | 5 | |
| LOW | RWKV-v1/src/utils.py | 4 | |
| LOW | RWKV-v1/src/trainer.py | 3 | |
| LOW | RWKV-v1/src/trainer.py | 6 | |
| LOW | RWKV-v1/src/trainer.py | 7 | |
| LOW | RWKV-v4neo/img_demoAE.py | 5 | |
| LOW | RWKV-v4neo/run.py | 6 | |
| LOW | RWKV-v4neo/train.py | 11 | |
| LOW | RWKV-v4neo/train.py | 135 | |
| LOW | RWKV-v4neo/train.py | 135 | |
| LOW | RWKV-v4neo/train.py | 135 | |
| LOW | RWKV-v4neo/src/model_run.py | 7 | |
| LOW | RWKV-v4neo/src/model_run.py | 10 | |
| LOW | RWKV-v4neo/src/model_run.py | 10 | |
| LOW | RWKV-v4neo/src/model.py | 12 | |
| LOW | RWKV-v4neo/src/model.py | 12 | |
| LOW | RWKV-v4neo/src/dataset.py | 5 | |
| LOW | RWKV-v4neo/src/dataset.py | 5 | |
| LOW | RWKV-v4neo/src/model_img.py | 6 | |
| LOW | RWKV-v4neo/src/model_img.py | 12 | |
| LOW | RWKV-v4neo/src/model_img.py | 12 | |
| LOW | RWKV-v4neo/src/model_img.py | 14 | |
| LOW | RWKV-v4neo/src/binidx.py | 1 | |
| LOW | RWKV-v4neo/src/binidx.py | 5 | |
| 64 more matches not shown… | |||
| Severity | File | Line | Snippet |
|---|---|---|---|
| LOW | RWKV-v7/rwkv_v7a_demo.py | 345 | |
| LOW | RWKV-v7/rwkv_v8_rc00_demo.py | 339 | |
| LOW | RWKV-v7/rwkv_v7_demo_fast.py | 334 | |
| LOW | RWKV-v7/rwkv_v7_demo_rnn.py | 234 | |
| LOW | RWKV-v7/rwkv_v7_demo.py | 88 | |
| LOW | RWKV-v7/rwkv_v7b_demo.py | 311 | |
| LOW | RWKV-v7/rwkv_v8_rc00_hybrid_demo.py | 295 | |
| LOW | RWKV-v7/train_temp/rwkv7_train_simplified.py | 67 | |
| LOW | RWKV-v7/train_temp/rwkv7_train_simplified.py | 89 | |
| LOW | RWKV-v7/train_temp/src/model.py | 499 | |
| LOW | RWKV-v7/train_temp/src/model.py | 894 | |
| LOW | RWKV-v7/train_temp/src/model.py | 526 | |
| LOW | RWKV-v7/train_temp/src/trainer.py | 159 | |
| LOW | RWKV-v7/train_temp/src/trainer.py | 130 | |
| LOW | RWKV-v8/251105_reverse_run.py | 41 | |
| LOW | RWKV-v8/251105_reverse_run.py | 145 | |
| LOW | RWKV-v8/251105_reverse_run.py | 167 | |
| LOW | RWKV-v8/251016_rosa_1bit_run.py | 9 | |
| LOW | RWKV-v8/251024_rosaQKV_run.py | 29 | |
| LOW | RWKV-v8/251024_rosaQKV_run.py | 133 | |
| LOW | RWKV-v8/251024_rosaQKV_run.py | 155 | |
| LOW | RWKV-v8/251014_rosa_onlyemb_train.py | 9 | |
| LOW | RWKV-v8/251014_rosa_1bit_layer.py | 6 | |
| LOW | RWKV-v8/251014_rosa_1bit_layer.py | 42 | |
| LOW | RWKV-v8/251014_rosa_1bit_train.py | 9 | |
| LOW | RWKV-v8/251014_rosa_1bit_train.py | 61 | |
| LOW | RWKV-v8/251018_rosa_4bit_run.py | 9 | |
| LOW | RWKV-v8/260222_rosa4bitLM_L12.py | 156 | |
| LOW | RWKV-v8/260222_rosa4bitLM_L12.py | 84 | |
| LOW | RWKV-v8/260222_rosa4bitLM_L12.py | 179 | |
| LOW | RWKV-v8/260212_rosa1bitLM_L12.py | 150 | |
| LOW | RWKV-v8/260212_rosa1bitLM_L12.py | 78 | |
| LOW | RWKV-v1/src/model.py | 16 | |
| LOW | RWKV-v1/src/model.py | 392 | |
| LOW | RWKV-v1/src/model.py | 461 | |
| LOW | RWKV-v1/src/trainer.py | 60 | |
| LOW | RWKV-v1/src/trainer.py | 65 | |
| LOW | RWKV-v4neo/chat.py | 197 | |
| LOW | RWKV-v4neo/math_demo/run.py | 41 | |
| LOW | RWKV-v4neo/src/model_run.py | 36 | |
| LOW | RWKV-v4neo/src/model.py | 747 | |
| LOW | RWKV-v4neo/src/model.py | 822 | |
| LOW | RWKV-v4neo/src/model.py | 913 | |
| LOW | RWKV-v4neo/src/dataset.py | 15 | |
| LOW | RWKV-v4neo/src/dataset.py | 107 | |
| LOW | RWKV-v4neo/src/model_img.py | 295 | |
| LOW | RWKV-v4neo/src/trainer.py | 203 | |
| LOW | RWKV-v4neo/src/trainer.py | 171 | |
| LOW | RWKV-v3/src/model_run.py | 182 | |
| LOW | RWKV-v3/src/model.py | 62 | |
| LOW | RWKV-v3/src/trainer.py | 78 | |
| LOW | RWKV-v3/src/trainer.py | 83 | |
| LOW | RWKV-v2-RNN/src/model_run.py | 14 | |
| LOW | RWKV-v2-RNN/src/model.py | 63 | |
| LOW | RWKV-v2-RNN/src/trainer.py | 78 | |
| LOW | RWKV-v2-RNN/src/trainer.py | 83 | |
| LOW | RWKV-v4/src/model_run.py | 211 | |
| LOW | RWKV-v4/src/model_run.py | 241 | |
| LOW | RWKV-v4/src/model.py | 107 | |
| LOW | RWKV-v4/src/model.py | 382 | |
| 15 more matches not shown… | |||
| Severity | File | Line | Snippet |
|---|---|---|---|
| HIGH | RWKV-v7/train_temp/src/model.py | 0 | ############################################################################ # # init model weight (slow for large model |
| HIGH | RWKV-v4neo/src/model.py | 0 | ############################################################################ # # init model weight (slow for large model |
| HIGH | RWKV-v4neo/src/model_img.py | 0 | ############################################################################ # # init model weight (slow for large model |
| HIGH | RWKV-v5/src/model.py | 0 | ############################################################################ # # init model weight (slow for large model |
| HIGH | RWKV-v7/train_temp/src/binidx.py | 0 | if distributed is initialized print only on rank 0. |
| HIGH | RWKV-v4neo/src/binidx.py | 0 | if distributed is initialized print only on rank 0. |
| HIGH | RWKV-v4/src/binidx.py | 0 | if distributed is initialized print only on rank 0. |
| HIGH | RWKV-v5/src/binidx.py | 0 | if distributed is initialized print only on rank 0. |
| HIGH | RWKV-v7/train_temp/src/binidx.py | 0 | retrieves a single item from the dataset with the option to only return a portion of the item. get(idx) is the same as [ |
| HIGH | RWKV-v4neo/src/binidx.py | 0 | retrieves a single item from the dataset with the option to only return a portion of the item. get(idx) is the same as [ |
| HIGH | RWKV-v4/src/binidx.py | 0 | retrieves a single item from the dataset with the option to only return a portion of the item. get(idx) is the same as [ |
| HIGH | RWKV-v5/src/binidx.py | 0 | retrieves a single item from the dataset with the option to only return a portion of the item. get(idx) is the same as [ |
| Severity | File | Line | Snippet |
|---|---|---|---|
| LOW | RWKV-v4neo/run.py | 20 | # Step 1: set model & config (use v4 to run your trained-from-scratch models. v4 and v4neo are compatible) |
| LOW | RWKV-v4neo/run.py | 78 | # Step 2: set prompt & sampling stuffs |
| LOW | RWKV-v3/run.py | 19 | ### Step 1: set model ################################################################################## |
| LOW | RWKV-v3/run.py | 37 | ### Step 2: set context ################################################################################ |
| LOW | RWKV-v3/train.py | 28 | ### Step 1: set training data ########################################################################## |
| LOW | RWKV-v3/train.py | 34 | ### Step 2: set model size ############################################################################# |
| LOW | RWKV-v3/train.py | 48 | ### Step 3: set batch size ############################################################################# |
| LOW | RWKV-v3/train.py | 55 | ### Step 4: set learning rate, number of mini-epochs ####################################################### |
| LOW | RWKV-v2-RNN/run.py | 20 | ### Step 1: set model ################################################################################## |
| LOW | RWKV-v2-RNN/run.py | 44 | ### Step 2: set context ################################################################################ |
| LOW | RWKV-v2-RNN/train.py | 17 | ### Step 1: set training data ########################################################################## |
| LOW | RWKV-v2-RNN/train.py | 23 | ### Step 2: set model size ############################################################################# |
| LOW | RWKV-v2-RNN/train.py | 32 | ### Step 3: set batch size ############################################################################# |
| LOW | RWKV-v2-RNN/train.py | 39 | ### Step 4: set learning rate, training mini-epochs ####################################################### |
| LOW | RWKV-v4/run.py | 19 | # Step 1: set model |
| LOW | RWKV-v4/run.py | 70 | # Step 2: set prompt & sampling stuffs |
| LOW | RWKV-v4/train.py | 22 | # Step 1: set training data & cfg |
| LOW | RWKV-v4/train.py | 78 | # Step 2: set model details |
| LOW | RWKV-v4/train.py | 110 | # Step 3: set batch size & learning rate etc. |
| Severity | File | Line | Snippet |
|---|---|---|---|
| LOW | RWKV-v7/train_temp/demo-training-run.sh | 1 | #!/bin/bash |
| LOW | RWKV-v7/train_temp/demo-training-run.sh | 21 | # |
| LOW | RWKV-v7/train_temp/demo-training-run-v7-pile.sh | 1 | #!/bin/bash |
| LOW | RWKV-v7/train_temp/demo-training-prepare-v7-pile.sh | 1 | #!/bin/bash |
| LOW | RWKV-v7/train_temp/train.py | 141 | rank_zero_info( |
| LOW | RWKV-v7/train_temp/demo-training-prepare.sh | 1 | #!/bin/bash |
| LOW | RWKV-v7/train_temp/src/model.py | 581 | self.key.weight.data.uniform_(-0.05/(C**0.5), 0.05/(C**0.5)) |
| LOW | RWKV-v7/train_temp/src/model.py | 621 | # v = v + (v_first - v) * torch.sigmoid(self.v0 + (xv @ self.v1) @ self.v2) # add value residual |
| LOW | RWKV-v7/train_temp/src/model.py | 641 | # kk = F.normalize(kk.view(B,T,H,-1), dim=-1, p=2.0).view(B,T,C) |
| LOW | RWKV-v7/train_temp/src/model.py | 681 | # super().__init__() |
| LOW | RWKV-v7/train_temp/src/model.py | 761 | # @staticmethod |
| LOW | RWKV-v4neo/run.py | 41 | # MODEL_NAME = "/fsx/BlinkDL/rwkv-release/RWKV-4-Pile-169M-20220807-8023" |
| LOW | RWKV-v4neo/run.py | 81 | # context = 'A' |
| LOW | RWKV-v4neo/train.py | 21 | # --data_file "" --data_type "dummy" --vocab_size 0 \ |
| LOW | RWKV-v4neo/train.py | 41 | # --micro_bsz 8 --n_layer 24 --n_embd 2048 --pre_ffn 0 --head_qk 0 \ |
| LOW | RWKV-v4neo/train.py | 261 | # |
| LOW | RWKV-v4neo/src/model.py | 881 | # if sum_mask == 0: |
| LOW | RWKV-v4neo/src/model_img.py | 421 | ss = n.split('.') |
| LOW | RWKV-v3/train.py | 41 | # 'RWKV' (better for English) or 'RWKV-ffnPre' (better in some cases) |
| LOW | RWKV-v4/train.py | 41 | |
| LOW | RWKV-v5/demo-training-run.sh | 1 | #!/bin/bash |
| LOW | RWKV-v5/demo-training-run.sh | 21 | # |
| LOW | RWKV-v5/rwkv_v6_demo.py | 221 | |
| LOW | RWKV-v5/rwkv_v6_demo.py | 241 | # att.output.weight => zero |
| LOW | RWKV-v5/demo-training-run-v7-pile.sh | 1 | #!/bin/bash |
| LOW | RWKV-v5/demo-training-prepare-v7-pile.sh | 1 | #!/bin/bash |
| LOW | RWKV-v5/train.py | 321 | data_loader = DataLoader(train_data, shuffle=False, pin_memory=True, batch_size=args.micro_bsz, num_workers=1, persi |
| LOW | RWKV-v5/demo-training-prepare.sh | 1 | #!/bin/bash |
| Severity | File | Line | Snippet |
|---|---|---|---|
| LOW | RWKV-v7/train_temp/demo-training-run.sh | 8 | # Therefore check the log (### Loading rwkv-xxx.pth... ###), and make sure you don't have extra rwkv-*.pth there |
| LOW | RWKV-v7/train_temp/demo-training-run-v7-pile.sh | 8 | # Therefore check the log (### Loading rwkv-xxx.pth... ###), and make sure you don't have extra rwkv-*.pth there |
| LOW | RWKV-v5/demo-training-run.sh | 8 | # Therefore check the log (### Loading rwkv-xxx.pth... ###), and make sure you don't have extra rwkv-*.pth there |
| LOW | RWKV-v5/demo-training-run-v7-pile.sh | 8 | # Therefore check the log (### Loading rwkv-xxx.pth... ###), and make sure you don't have extra rwkv-*.pth there |
| Severity | File | Line | Snippet |
|---|---|---|---|
| LOW | RWKV-v4/run.py | 21 | # Set TOKEN_MODE to 'char' or 'bpe' if the model is trained by 'train.py' from scratch. |
| LOW | RWKV-v4/run.py | 23 | # Set TOKEN_MODE to 'pile' if you want to test pre-trained pile models. |
| Severity | File | Line | Snippet |
|---|---|---|---|
| LOW | RWKV-v3/run.py | 32 | UNKNOWN_CHAR = ' ' # here we just set it to [space] for simplicity |
| LOW | RWKV-v2-RNN/run.py | 39 | UNKNOWN_CHAR = ' ' # here we just set it to [space] for simplicity |
| LOW | RWKV-v4/run.py | 36 | UNKNOWN_CHAR = ' ' # here we just set it to ' ' for simplicity |
| LOW | RWKV-v4/verify.py | 28 | UNKNOWN_CHAR = ' ' # here we just set it to [space] for simplicity |
| Severity | File | Line | Snippet |
|---|---|---|---|
| LOW | RWKV-v7/train_temp/src/trainer.py | 148 | except Exception as e: |
| LOW | RWKV-v4neo/src/trainer.py | 189 | except Exception as e: |
| LOW | RWKV-v5/src/trainer.py | 196 | except Exception as e: |