InternLM

@misc{2023internlm,
    title={InternLM: A Multilingual Language Model with Progressively Enhanced Capabilities},
    author={InternLM Team},
    howpublished = {\url{https://github.com/InternLM/InternLM}},
    year={2023}
}

# 请安装torch=2.0.0和transformers=4.30.2版本:
# pip install torch==2.0.0 -i https://pypi.tuna.tsinghua.edu.cn/simple
# pip install transformers==4.30.2 -i https://pypi.tuna.tsinghua.edu.cn/simple
python ./research/internlm/convert_weight.py --torch_ckpt_path TORCH_CKPT_PATH --mindspore_ckpt_path MS_CKPT_NAME
# 参数说明
torch_ckpt_path: Hugging Face权重保存目录路径下任意权重bin文件，将根据该文件路径读取目录下全部权重
mindspore_ckpt_path: 转换后MindSpore权重文件的保存路径

# research/internlm/predict_internlm_7b.yaml
# runner config
runner_config:
  epochs: 1
  batch_size: 1                 # batch_size设为1
  sink_mode: True
  sink_size: 2
...
# model config
model:
  model_config:
    type: LlamaConfig
    ...
    rms_norm_eps: 1.0e-6        # rms_norm_eps设为1.0e-6
...
processor:
 return_tensors: ms
 tokenizer:
   unk_token: '<unk>'
   bos_token: '<s>'
   eos_token: '</s>'
   pad_token: '</s>'
   vocab_file: '/path/Internlm-7b/tokenizer.model'        # 添加tokenizer路径
   type: InternLMTokenizer

python run_internlm.py \
--config "predict_internlm_7b.yaml" \
--run_mode predict \
--use_parallel False \
--load_checkpoint ckpt_path_or_dir \
--predict_data '我们来对对联吧！生意如春意 的下联是' \
--device_id 0

# [{'text_generation_text': ['<|User|>:我们来对对联吧！生意如春意的下联是\n<|Bot|>:生意如春意\n']}]

from mindspore import context
from mindformers.pipeline import pipeline

from internlm import InternLMForCausalLM
from internlm_tokenizer import InternLMTokenizer
from internlm_config import InternLMConfig

context.set_context(device_id=0, mode=0)
# init model
internlm_model_path = "/path/InternLM-7B/internlm-chat.ckpt" # InternLM ckpt path
internlm_config = InternLMConfig(
    checkpoint_name_or_path=internlm_model_path,
    use_past=True,
    is_dynamic=True
)
internlm_model = InternLMForCausalLM(
    config=internlm_config
)
# init tokenizer
tokenizer_path = "/path/InternLM-7B/tokenizer.model" # InternLM-7B tokenizer.model path
tokenizer = InternLMTokenizer(
    vocab_file=tokenizer_path
)
pipeline_task = pipeline(task="text_generation", model=internlm_model, tokenizer=tokenizer)
pipeline_result = pipeline_task("<s><|User|>:你好<eoh>\n<|Bot|>:",
                                do_sample=False,
                                repetition_penalty=1.0,
                                max_length=256)

print(pipeline_result)

# [{'text_generation_text': ['<|User|>:你好\n<|Bot|>:你好，有什么我可以帮助你的吗？\n']}]

python wiki_data_preprocess.py \
--mindrecord_schema internlm_wiki \
--input_glob {path}/wikitext-2/wiki.train.tokens \
--output_file {path}/wiki_processed/wiki.mindrecord \
--model_file {path}/tokenizer.model \
--seq_length 2048 \
--min_length 50  # 过滤token长度小于min_length的数据，default=50

python alpaca_data_preprocess.py \
--mindrecord_schema internlm_alpaca \
--input_glob {path}/alpaca_data.json \
--output_file {path}/alpaca_processed/alpaca.mindrecord \
--model_file {path}/tokenizer.model \
--seq_length 2048

    └── path of ckpt
        └── rank_0
            └── internlm_7b_base.ckpt

output_dir: './output'             # path to save checkpoint/strategy
load_checkpoint: 'path/of/ckpt'    # 添加预训练权重路径
auto_trans_ckpt: True              # 开启权重自动切分
only_save_strategy: False
resume_training: False
use_parallel: True
run_mode: 'finetune'
# dataset
train_dataset: &train_dataset
  data_loader:
    type: MindDataset
    dataset_dir: "{path}/alpaca.mindrecord"   # 修改训练数据集路径
    shuffle: True
  input_columns: ["input_ids", "labels"]
# 用alpaca数据集指令微调时，input_columns: ["input_ids", "labels"]
# 用wiki数据集微调时，input_columns: ["input_ids"]

bash scripts/msrun_launcher.sh \
"python research/internlm/run_internlm.py \
--run_mode finetune \
--use_parallel True \
--config research/internlm/finetune_internlm_7b.yaml \
--load_checkpoint path/to/ckpt \
--auto_trans_ckpt True \
--train_dataset {path}/train_data" 8

python run_internlm.py \
--config finetune_internlm_7b_lora.yaml \
--run_mode finetune \
--use_parallel False \
--load_checkpoint path/to/ckpt \
--auto_trans_ckpt False \
--train_dataset {path}/train_data \
--device_id 0

bash scripts/msrun_launcher.sh \
"python research/internlm/run_internlm.py \
--run_mode finetune \
--use_parallel True \
--config research/internlm/finetune_internlm_7b_lora.yaml \
--load_checkpoint path/to/ckpt \
--auto_trans_ckpt True \
--train_dataset {path}/train_data" 8

auto_trans_ckpt: False                              # 关闭自动权重转换
use_past: True                                      # 使用增量推理
vocab_file: '/path/to/tokenizer.model'              # 配置词表路径

python run_internlm.py \
--config 'predict_internlm_20b.yaml' \
--run_mode predict \
--use_parallel False \
--load_checkpoint '/path/to/InternLM-20B-Chat.ckpt' \
--predict_data '你是谁？' \
--device_id 0

# output: [{'text_generation_text': ['<|User|>:你是谁？<eoh>\n<|Bot|>:我是一个名叫书生·浦语的人工智能助手，由上海人工智能实验室开发。我能够回答问题、提供定义和解释、将文本从一种语言翻译成另一种语言、总结文本、生成文本、编写故事、分析情感、提供推荐、开发算法、编写代码以及其他任何基于语言的任务。我的设计理念是有用、诚实并且无害。我使用了Transformer模型和深度学习技术，并使用语言模型作为预训练任务。我能够使用汉语和英语进行交流。<eoa>\n']}]

# run_internlm_pipeline.py
import mindspore as ms
from mindspore import context
from mindformers.pipeline import pipeline

from mindformers import MindFormerConfig
from internlm import InternLMForCausalLM
from internlm_config import InternLMConfig
from internlm_tokenizer import InternLMTokenizer


# init context
context.set_context(device_id=0, mode=0)

# init config
internlm_config_path = "/path/to/predict_internlm_20b.yaml"
config = MindFormerConfig(internlm_config_path)
internlm_config = InternLMConfig(**config.model.model_config)
internlm_model = InternLMForCausalLM(
    config=internlm_config
)

# init tokenizer
tokenizer_path = "/path/to/InternLM-20B/tokenizer.model" # InternLM-20B tokenizer path
tokenizer = InternLMTokenizer(
    vocab_file=tokenizer_path
)

# init and run pipeline
pipeline_task = pipeline(task="text_generation", model=internlm_model, tokenizer=tokenizer)
pipeline_result = pipeline_task("<s><|User|>:你是谁？<eoh>\n<|Bot|>:",
                                do_sample=False,
                                repetition_penalty=1.0,
                                max_length=256)
print(pipeline_result)

# 推理输出
# [{'text_generation_text': ['<|User|>:你是谁？<eoh>\n<|Bot|>:我是一个名叫书生·浦语的人工智能助手，由上海人工智能实验室开发。我能够回答问题、提供定义和解释、将文本从一种语言翻译成另一种语言、总结文本、生成文本、编写故事、分析情感、提供推荐、开发算法、编写代码以及其他任何基于语言的任务。我的设计理念是有用、诚实并且无害。我使用了Transformer模型和深度学习技术，并使用语言模型作为预训练任务。我能够使用汉语和英语进行交流。<eoa>\n']}]

load_checkpoint: ''                                           # 单卡推理时，只需配置checkpoint_name_or_path
auto_trans_ckpt: False                                        # 关闭自动权重转换
checkpoint_name_or_path: '/path/to/InternLM-20B-Chat.ckpt'    # 填写权重绝对路径
use_past: True                                                # 使用增量推理
is_dynamic: True                                              # 使用动态shape
block_size: 16                                                # Paged Attention中每个block的大小
num_blocks: 512                                               # Paged Attention中的block数量
vocab_file: '/path/to/tokenizer.model'                        # 配置词表路径
use_parallel: False                                           # 关闭并行模式

python internlm/run_internlm_pipeline.py

# run_internlm_generate.py
import mindspore as ms
from mindspore import context

from mindformers import MindFormerConfig
from internlm import InternLMForCausalLM
from internlm_config import InternLMConfig
from internlm_tokenizer import InternLMTokenizer


# init context
context.set_context(device_id=0, mode=0)

# init config
internlm_config_path = "/path/to/predict_internlm_20b.yaml"
config = MindFormerConfig(internlm_config_path)
internlm_config = InternLMConfig(**config.model.model_config)
internlm_model = InternLMForCausalLM(
    config=internlm_config
)

# init tokenizer
tokenizer_path = "/path/to/InternLM-20B/tokenizer.model" # InternLM-20B tokenizer path
tokenizer = InternLMTokenizer(
    vocab_file=tokenizer_path
)

# predict using generate
input_ids = tokenizer("<s><|User|>:你是谁？<eoh>\n<|Bot|>:",
                      max_length=64, padding="max_length")["input_ids"]
generate_ids = internlm_model.generate(input_ids,
                                       do_sample=False,
                                       top_k=1,
                                       top_p=1.0,
                                       repetition_penalty=1.0,
                                       temperature=1.0,
                                       max_length=256)
generate_result = tokenizer.decode(generate_ids)
print(generate_result)

# 推理输出
# [{'text_generation_text': ['<|User|>:你是谁？<eoh>\n<|Bot|>:我是一个名叫书生·浦语的人工智能助手，由上海人工智能实验室开发。我使用了Transformer模型和深度学习技术，并使用语言模型作为预训练任务。我的设计理念是有用、诚实并且无害。我能够回答问题、提供定义和解释、将文本从一种语言翻译成另一种语言、总结文本、生成文本、编写故事、分析情感、提供推荐、开发算法、编写代码以及其他任何基于语言的任务。但我不能看、听、尝、触摸、闻、移动、与物理世界交互、感受情感或体验感官输入、执行需要身体能力的任务。<eoa>\n']}]

load_checkpoint: ''                                           # 单卡推理时，只需配置checkpoint_name_or_path
auto_trans_ckpt: False                                        # 关闭自动权重转换
checkpoint_name_or_path: '/path/to/InternLM-20B-Chat.ckpt'    # 填写权重绝对路径
use_past: True                                                # 使用增量推理
is_dynamic: True                                              # 使用动态shape
block_size: 16                                                # Paged Attention中每个block的大小
num_blocks: 512                                               # Paged Attention中的block数量
vocab_file: '/path/to/tokenizer.model'                        # 配置词表路径
use_parallel: False                                           # 关闭并行模式

python internlm/run_internlm_generate.py