Question 1

Figure from the original question paper Consider the following code where a tokenizer is created using the BPE model: from tokenizers import Tokenizer from tokenizers.models import BPE from tokenizers.trainers import BpeTrainer from tokenizers.normalizers import Lowercase from tokenizers.pre_tokenizers import Whitespace data = ["existent", "non", "exist", "word"] # Create a tokenizer with BPE model without specifying the unk_token model = BPE() tokenizer = Tokenizer(model) # Normalizer and Pre-tokenizer tokenizer.normalizer = Lowercase() tokenizer.pre_tokenizer = Whitespace() # Train the tokenizer trainer = BpeTrainer(vocab_size=5000, special_tokens=["<s>", "</s>", "<pad>"]) tokenizer.train_from_iterator(data, trainer) # Access the tokenizer's vocabulary vocab = tokenizer.get_vocab() # Test tokenization with a word not present in the vocabulary tokens = tokenizer.encode("nonexistentword").tokens print("Tokens:", tokens) Which of the following statements is correct about the tokenization output when the word "nonexistentword" is encountered? Consider the following code snippet: from tokenizers import Tokenizer from tokenizers.models import BPE from tokenizers.trainers import BpeTrainer from tokenizers.normalizers import Lowercase from tokenizers.pre_tokenizers import Whitespace from tokenizers.processors import TemplateProcessing data = ["hello", "world"] # Create a tokenizer with BPE model model = BPE() tokenizer = Tokenizer(model) # Normalizer and Pre-tokenizer tokenizer.normalizer = Lowercase() tokenizer.pre_tokenizer = Whitespace() # Trainer for the tokenizer trainer = BpeTrainer(vocab_size=5000, special_tokens=["<s>", "</s>", "<pad>", "<unk>"]) tokenizer.train_from_iterator(data, trainer) # Post-processor tokenizer.post_processor = TemplateProcessing(single="[CLS] $0 [SEP]", special_tokens=[("[CLS]", 2), ("[SEP]", 3)]) # Encode input and get the token IDs encoded = tokenizer.encode("Hello world") print("Token IDs:", encoded.ids) What will likely be printed as the output?