## Commands
### k-shot
#### snips
##### en-en
CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint gpt2 --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 1


CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint t5-large --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 1

### running benchmark

### SNIPS
bash ./scripts/run_log_prob_custom_confidence.sh 0 snips openai-gpt value en en 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 snips gpt2 value en en 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 snips gpt2-medium value en en 42
bash ./scripts/run_log_prob_custom_confidence.sh 8 snips gpt2-large value en en 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 snips gpt2-xl value en en 42

bash ./scripts/run_log_prob_custom_confidence.sh 1 snips EleutherAI/gpt-neo-1.3B value en en 42


CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint gpt2-xl --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 20

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint gpt2 --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 20

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint gpt2-medium --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 20

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint gpt2-large --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 20

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint openai-gpt --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 20

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 20

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 30

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset snips --model_checkpoint gpt-j --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 60

### MTOP
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2 value de de 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2 value en en 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2 value en es 42
bash ./scripts/run_log_prob_custom_confidence.sh 2 mtop gpt2 value es es 42
bash ./scripts/run_log_prob_custom_confidence.sh 2 mtop gpt2 value es en 42
bash ./scripts/run_log_prob_custom_confidence.sh 3 mtop gpt2 value fr en 42
bash ./scripts/run_log_prob_custom_confidence.sh 3 mtop gpt2 value fr fr 42

bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2 value de en 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2 value de es 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 2 mtop gpt2 value de fr 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 3 mtop gpt2 value en de 42 
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2 value en fr 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2 value es de 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 2 mtop gpt2 value es fr 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 3 mtop gpt2 value fr de 42 
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2 value fr es 42

bash ./scripts/run_log_prob_custom_confidence.sh 3 mtop gpt2-medium value de de 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-medium value en en 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-medium value es es 42
bash ./scripts/run_log_prob_custom_confidence.sh 2 mtop gpt2-medium value fr fr 42
bash ./scripts/run_log_prob_custom_confidence.sh 4 mtop gpt2-medium value de en 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 8 mtop gpt2-medium value de es 42 & 
bash ./scripts/run_log_prob_custom_confidence.sh 8 mtop gpt2-medium value en de 42  
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-medium value en es 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-medium value en fr 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 2 mtop gpt2-medium value es de 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 3 mtop gpt2-medium value es en 42  
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-medium value es fr 42  
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-medium value de fr 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-medium value fr de 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 2 mtop gpt2-medium value fr en 42 & 
bash ./scripts/run_log_prob_custom_confidence.sh 3 mtop gpt2-medium value fr es 42 & 

bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-large value de de 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value es es 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-large value en en 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value fr fr 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value en fr 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-large value en de 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-large value en es 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-large value de es 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-large value de en 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-large value de fr 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value es de 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value es en 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value es fr 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value fr en 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value fr es 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value fr de 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value en fr 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value en de 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value en es 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 mtop gpt2-large value de es 42

bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-xl value de de 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-xl value es es 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-xl value en en 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-xl value fr fr 42

CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset mtop --model_checkpoint gpt2-xl --cuda --length 5 --label_type value --src_lang de --tgt_lang de --seed 42 --use_log_prob --use_confidence --k_shot 10

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset mtop --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang de --tgt_lang de --seed 42 --use_log_prob --use_confidence --k_shot 5

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset mtop --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang fr --tgt_lang fr --seed 42 --use_log_prob --use_confidence --k_shot 5

CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset mtop --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 5

CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset mtop --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang es --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 5

bash ./scripts/run_log_prob_custom_confidence.sh 0 mtop gpt2-xl value en de 42

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset mtop --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 20 --fp16
CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset mtop --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 30 --fp16
<PENDING>
CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset mtop --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 40 --fp16

# MultiNLU
bash ./scripts/run_log_prob_custom_confidence.sh 3 multi-nlu openai-gpt value en en 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 0 multi-nlu openai-gpt value es es 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 0 multi-nlu openai-gpt value es en 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 3 multi-nlu openai-gpt value en es 42 &

bash ./scripts/run_log_prob_custom_confidence.sh 0 multi-nlu gpt2 value en en 42
bash ./scripts/run_log_prob_custom_confidence.sh 1 multi-nlu gpt2 value es es 42
bash ./scripts/run_log_prob_custom_confidence.sh 2 multi-nlu gpt2 value en es 42
bash ./scripts/run_log_prob_custom_confidence.sh 3 multi-nlu gpt2 value es en 42

bash ./scripts/run_log_prob_custom_confidence.sh 0 multi-nlu gpt2-medium value en en 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 1 multi-nlu gpt2-medium value es es 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 2 multi-nlu gpt2-medium value en es 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 3 multi-nlu gpt2-medium value es en 42 &

bash ./scripts/run_log_prob_custom_confidence.sh 1 multi-nlu gpt2-large value en en 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 2 multi-nlu gpt2-large value es es 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 3 multi-nlu gpt2-large value en es 42 &
bash ./scripts/run_log_prob_custom_confidence.sh 0 multi-nlu gpt2-large value es en 42 &

bash ./scripts/run_log_prob_custom_confidence.sh 0 multi-nlu gpt2-xl value es es 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 multi-nlu gpt2-xl value en es 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 multi-nlu gpt2-xl value es en 42
bash ./scripts/run_log_prob_custom_confidence.sh 0 multi-nlu gpt2-xl value en en 42

python evaluate.py  --dataset multi-nlu --model_checkpoint gpt2-xl --cuda --length 5 --label_type value --src_lang es --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 10

CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang es --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 1
CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 1
CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang es --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 5
CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 5

python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-2.7B --cuda --length 5 --label_type value --src_lang es --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 1
python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-2.7B --cuda --length 5 --label_type value --src_lang es --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 5
python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-2.7B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 1
python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-2.7B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 5
python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-2.7B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 10


python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-2.7B --cuda --length 5 --label_type value --src_lang es --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 10

CUDA_VISIBLE_DEVICES=0 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-2.7B --cuda --length 5 --label_type value --src_lang en --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 10

CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 10

bash ./scripts/run_log_prob_custom_confidence.sh 1 multi-nlu EleutherAI/gpt-neo-1.3B value es en 42

CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang es --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 20
CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 20
CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-2.7B --cuda --length 5 --label_type value --src_lang es --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 20
CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-2.7B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 20

<PENDING>
python evaluate.py  --dataset multi-nlu --model_checkpoint gpt2-xl --cuda --length 5 --label_type value --src_lang es --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 8
python evaluate.py  --dataset multi-nlu --model_checkpoint gpt2-xl --cuda --length 5 --label_type value --src_lang es --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 9
python evaluate.py  --dataset multi-nlu --model_checkpoint gpt2-xl --cuda --length 5 --label_type value --src_lang es --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 8
python evaluate.py  --dataset multi-nlu --model_checkpoint gpt2-xl --cuda --length 5 --label_type value --src_lang es --tgt_lang es --seed 42 --use_log_prob --use_confidence --k_shot 9



deepspeed --include=localhost:0,1,2,4,5,6,7,8,9 evaluate_deepspeed.py --dataset snips --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 10 --deepspeed --fp16 --num_gpus 9

deepspeed --include=localhost:1 evaluate_deepspeed.py --dataset multi-nlu --model_checkpoint EleutherAI/gpt-neo-1.3B --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 10 --deepspeed --fp16 --num_gpus 1




deepspeed --include=localhost:0,1,2,3 evaluate_deepspeed.py --dataset mtop --model_checkpoint gpt2-xl --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 10 --deepspeed --fp16 --num_gpus 4





* pip install -U git+https://github.com/microsoft/DeepSpeed.git@c697d7ae1cf5a479a8a85afa3bf9443e7d54ac2b

## examples
### Monolingual (en)
python train_debug.py --model_checkpoint gpt2 --cuda --num_return_sequences 1 --length 10 --label_type value-binary --k_shot 3 --src_lang en --tgt_lang en

```
PREFIX
('Remind me to text mom the address to the party tonight. => intent=create_reminder\nAdd driver license renewal to my errands reminder tomorrow => intent=None\nRemind Caitlyn to collect our mail for the next two weeks. => intent=create_reminder\nlist friends in Portland => intent=None\nRemind me to pick up orange juice after the movie today => intent=create_reminder\nWhere does Robert born? => intent=None\nRemind me at 8pm to  watch Orange Is The New Black => intent=',)

PRED
['create_reminder']
```

```
PREFIX
('Switch to new incoming call => intent=switch_call\nthe ingredients for sloppy joe => intent=None\nleave and go to new call => intent=switch_call\nCreate alarms 8pm and 11pm today => intent=None\nplease leave and switch calls => intent=switch_call\nWhat is the temperature going to be tommorrow? => intent=None\nhold and receive new call => intent=',)

PRED
['switch_call']
```

python train_debug.py --model_checkpoint gpt2 --cuda --num_return_sequences 1 --length 10 --label_type value-ori --k_shot 3 --src_lang en --tgt_lang en

```
PREFIX
('Remind me to text mom the address to the party tonight. => intent=create_reminder\nAdd driver license renewal to my errands reminder tomorrow => intent=update_reminder_todo\nRemind Caitlyn to collect our mail for the next two weeks. => intent=create_reminder\nlist friends in Portland => intent=get_contact\nRemind me to pick up orange juice after the movie today => intent=create_reminder\nWhere does Robert born? => intent=get_location\nRemind me at 8pm to  watch Orange Is The New Black => intent=',)

PRED
['create_reminder']
```

```
PREFIX
('Switch to new incoming call => intent=switch_call\nthe ingredients for sloppy joe => intent=get_info_recipes\nleave and go to new call => intent=switch_call\nCreate alarms 8pm and 11pm today => intent=create_alarm\nplease leave and switch calls => intent=switch_call\nWhat is the temperature going to be tommorrow? => intent=get_weather\nhold and receive new call => intent=',)

PRED
['switch_call']
```

### Cross-lingual (de-en)
python train_debug.py --model_checkpoint gpt2 --cuda --num_return_sequences 1 --length 10 --label_type value-binary --k_shot 3 --src_lang de --tgt_lang en

```
PREFIX
('Was findet heute Abend in Chicago statt? => intent=get_event\nStarte einen 15-Minuten-Timer. => intent=None\nWas ist diese Woche in LA los? => intent=get_event\nkürzlich angesehene Beiträge abrufen => intent=None\nWas ist Samstagabend in Pensacola los => intent=get_event\nStelle einen Wecker für 5:55 Uhr und 6:00 Uhr diesen Donnerstag => intent=None\nShow me dates for music festivals in 2018 => intent=',)

PRED
['get_event']
```

```
PREFIX
('Was ist diese Nacht die niedrigste Temperatur? => intent=get_weather\nWer arbeitet bei TCBY? => intent=None\nWetter in Südamerika im November => intent=get_weather\nErstell eine einstündige Trainingseinheit. => intent=None\nNenn mir die Vorhersage für Russland => intent=get_weather\nWer arbeitet bei Napa? => intent=None\nWhat does the forecast look like for the week => intent=',)

PRED
['get_weather']
```

python train_debug.py --model_checkpoint gpt2 --cuda --num_return_sequences 1 --length 10 --label_type value-ori --k_shot 3 --src_lang de --tgt_lang en

```
PREFIX
('Was findet heute Abend in Chicago statt? => intent=get_event\nStarte einen 15-Minuten-Timer. => intent=create_timer\nWas ist diese Woche in LA los? => intent=get_event\nkürzlich angesehene Beiträge abrufen => intent=get_stories_news\nWas ist Samstagabend in Pensacola los => intent=get_event\nStelle einen Wecker für 5:55 Uhr und 6:00 Uhr diesen Donnerstag => intent=create_alarm\nShow me dates for music festivals in 2018 => intent=',)

PRED
['get_events']
```

```
PREFIX
('Was ist diese Nacht die niedrigste Temperatur? => intent=get_weather\nWer arbeitet bei TCBY? => intent=get_contact\nWetter in Südamerika im November => intent=get_weather\nErstell eine einstündige Trainingseinheit. => intent=create_timer\nNenn mir die Vorhersage für Russland => intent=get_weather\nWer arbeitet bei Napa? => intent=get_contact\nWhat does the forecast look like for the week => intent=',)

PRED
['get_weather']
```

## T5 exmaple
CUDA_VISIBLE_DEVICES=1 python evaluate.py  --dataset snips --model_checkpoint t5-3b --cuda --length 5 --label_type value --src_lang en --tgt_lang en --seed 42 --use_log_prob --use_confidence --k_shot 5 --is_t5 --fp16