NLP with Hugging Face Transformers

  pip install transformers torch sentencepiece
  

  from transformers import pipeline

classifier = pipeline("sentiment-analysis")

results = classifier([
    "I love this product!",
    "Terrible experience, would not recommend.",
    "It was okay, nothing special.",
])

for text, result in zip(
    ["I love...", "Terrible...", "It was okay..."],
    results,
):
    print(f"{result['label']}: {result['score']:.3f}")
  

  from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

model_name = "distilbert-base-uncased-finetuned-sst-2-english"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name)

def classify(text):
    inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512)
    with torch.no_grad():
        outputs = model(**inputs)
    probs = torch.softmax(outputs.logits, dim=1)
    label = model.config.id2label[probs.argmax().item()]
    confidence = probs.max().item()
    return label, confidence

label, conf = classify("This movie was absolutely fantastic!")
print(f"{label} ({conf:.2f})")
  

  ner = pipeline("ner", grouped_entities=True)

text = "Apple Inc. was founded by Steve Jobs in Cupertino, California."
entities = ner(text)

for entity in entities:
    print(f"{entity['word']}: {entity['entity_group']} ({entity['score']:.2f})")
  

  generator = pipeline("text-generation", model="gpt2")

prompt = "Python is a programming language that"
output = generator(prompt, max_length=50, num_return_sequences=1)
print(output[0]["generated_text"])
  

  qa = pipeline("question-answering")

context = """
Python was created by Guido van Rossum and first released in 1991.
It emphasizes code readability and supports multiple programming paradigms.
"""

result = qa(question="Who created Python?", context=context)
print(f"Answer: {result['answer']} (confidence: {result['score']:.2f})")
  

  from transformers import TrainingArguments, Trainer
from datasets import load_dataset

dataset = load_dataset("imdb")  # movie reviews

training_args = TrainingArguments(
    output_dir="./results",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    eval_strategy="epoch",
    logging_steps=100,
)

# Define model, tokenizer, data collator, then:
# trainer = Trainer(model=model, args=training_args, train_dataset=..., eval_dataset=...)
# trainer.train()