Gaze/ChatGLM2-6B
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix max sequence padding

Browse Source 
Use transformers trainer
This commit is contained in:
duzx16 2023-07-04 20:57:14 +08:00
parent b99e3d74c9
commit a33cb4f8b9
4 changed files with 16 additions and 3776 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ptuning/evaluate.sh
View File

@ -10,7 +10,7 @@ torchrun --standalone --nnodes=1 --nproc-per-node=$NUM_GPUS main.py \
--overwrite_cache \ --overwrite_cache \
--prompt_column content \ --prompt_column content \
--response_column summary \ --response_column summary \
--model_name_or_path chatglm2-6b \ --model_name_or_path THUDM/chatglm2-6b \
--ptuning_checkpoint ./output/$CHECKPOINT/checkpoint-$STEP \ --ptuning_checkpoint ./output/$CHECKPOINT/checkpoint-$STEP \
--output_dir ./output/$CHECKPOINT \ --output_dir ./output/$CHECKPOINT \
--overwrite_output_dir \ --overwrite_output_dir \

4
ptuning/main.py
View File

@ -178,7 +178,7 @@ def main():
return model_inputs return model_inputs
def preprocess_function_train(examples): def preprocess_function_train(examples):
max_seq_length = data_args.max_source_length + data_args.max_target_length max_seq_length = data_args.max_source_length + data_args.max_target_length + 1
model_inputs = { model_inputs = {
"input_ids": [], "input_ids": [],
@ -335,7 +335,7 @@ def main():
tokenizer=tokenizer, tokenizer=tokenizer,
data_collator=data_collator, data_collator=data_collator,
compute_metrics=compute_metrics if training_args.predict_with_generate else None, compute_metrics=compute_metrics if training_args.predict_with_generate else None,
save_prefixencoder=model_args.pre_seq_len is not None save_changed=model_args.pre_seq_len is not None
) )
# Training # Training

3782
ptuning/trainer.py
View File

File diff suppressed because it is too large Load Diff

4
ptuning/trainer_seq2seq.py
View File

@ -19,7 +19,7 @@ from torch import nn
from torch.utils.data import Dataset from torch.utils.data import Dataset
from transformers.deepspeed import is_deepspeed_zero3_enabled from transformers.deepspeed import is_deepspeed_zero3_enabled
from trainer import Trainer from trainer import PrefixTrainer
from transformers.trainer_utils import PredictionOutput from transformers.trainer_utils import PredictionOutput
from transformers.utils import logging from transformers.utils import logging
@ -27,7 +27,7 @@ from transformers.utils import logging
logger = logging.get_logger(__name__) logger = logging.get_logger(__name__)
class Seq2SeqTrainer(Trainer): class Seq2SeqTrainer(PrefixTrainer):
def evaluate( def evaluate(
self, self,
eval_dataset: Optional[Dataset] = None, eval_dataset: Optional[Dataset] = None,