Transformers : l'Architecture qui a Tout Changé

Attention(Q, K, V) = softmax(Q · Kᵀ / √dₖ) · V

import torch
import torch.nn.functional as F

def self_attention(x, d_model=64):
    """
    x : tensor de shape (batch, seq_len, d_model)
    """
    # Projections linéaires
    W_q = torch.nn.Linear(d_model, d_model, bias=False)
    W_k = torch.nn.Linear(d_model, d_model, bias=False)
    W_v = torch.nn.Linear(d_model, d_model, bias=False)

    Q = W_q(x)  # (batch, seq_len, d_model)
    K = W_k(x)
    V = W_v(x)

    # Score d''attention
    d_k = Q.size(-1)
    scores = torch.matmul(Q, K.transpose(-2, -1)) / (d_k ** 0.5)

    # Softmax → poids d''attention
    attn_weights = F.softmax(scores, dim=-1)

    # Sortie pondérée
    output = torch.matmul(attn_weights, V)
    return output, attn_weights

class MultiHeadAttention(torch.nn.Module):
    def __init__(self, d_model=512, n_heads=8):
        super().__init__()
        self.n_heads = n_heads
        self.d_head = d_model // n_heads

        self.W_q = torch.nn.Linear(d_model, d_model)
        self.W_k = torch.nn.Linear(d_model, d_model)
        self.W_v = torch.nn.Linear(d_model, d_model)
        self.W_o = torch.nn.Linear(d_model, d_model)

    def forward(self, x):
        batch, seq_len, d_model = x.shape

        # Projeter et diviser en têtes
        Q = self.W_q(x).view(batch, seq_len, self.n_heads, self.d_head).transpose(1, 2)
        K = self.W_k(x).view(batch, seq_len, self.n_heads, self.d_head).transpose(1, 2)
        V = self.W_v(x).view(batch, seq_len, self.n_heads, self.d_head).transpose(1, 2)

        # Attention par tête
        scores = torch.matmul(Q, K.transpose(-2, -1)) / (self.d_head ** 0.5)
        attn = F.softmax(scores, dim=-1)
        context = torch.matmul(attn, V)

        # Recombiner les têtes
        context = context.transpose(1, 2).contiguous().view(batch, seq_len, d_model)
        return self.W_o(context)

import math

def positional_encoding(seq_len, d_model):
    pe = torch.zeros(seq_len, d_model)
    position = torch.arange(0, seq_len).unsqueeze(1).float()
    div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))

    pe[:, 0::2] = torch.sin(position * div_term)  # Dimensions paires
    pe[:, 1::2] = torch.cos(position * div_term)  # Dimensions impaires
    return pe

class TransformerBlock(torch.nn.Module):
    def __init__(self, d_model=512, n_heads=8, d_ff=2048, dropout=0.1):
        super().__init__()
        self.attention = MultiHeadAttention(d_model, n_heads)
        self.norm1 = torch.nn.LayerNorm(d_model)
        self.norm2 = torch.nn.LayerNorm(d_model)
        self.ffn = torch.nn.Sequential(
            torch.nn.Linear(d_model, d_ff),
            torch.nn.ReLU(),
            torch.nn.Linear(d_ff, d_model),
        )
        self.dropout = torch.nn.Dropout(dropout)

    def forward(self, x):
        # Self-attention + residual
        attn_out = self.attention(x)
        x = self.norm1(x + self.dropout(attn_out))

        # Feed-forward + residual
        ffn_out = self.ffn(x)
        x = self.norm2(x + self.dropout(ffn_out))
        return x

Entrée :  "Le [MASK] dort sur le tapis"
BERT :    "Le  chat   dort sur le tapis"

Entrée :  "Le chat dort sur"
GPT :     → "le"  → "tapis"  → "."

from transformers import pipeline

# Charger un pipeline de classification de sentiment
classifier = pipeline("sentiment-analysis", model="nlptown/bert-base-multilingual-uncased-sentiment")

# Tester sur des phrases françaises
phrases = [
    "Ce film est absolument magnifique, je le recommande !",
    "Service client horrible, jamais plus.",
    "Le repas était correct, sans plus.",
]

for phrase in phrases:
    result = classifier(phrase)[0]
    print(f"'{phrase[:50]}...' → {result['label']} ({result['score']:.2f})")

from transformers import pipeline

generator = pipeline("text-generation", model="gpt2")

prompt = "Deep learning is"
results = generator(prompt, max_length=50, num_return_sequences=3)

for i, result in enumerate(results):
    print(f"\n--- Génération {i+1} ---")
    print(result["generated_text"])