Just uploading entire rewritten codebase at once

attention.py

@@ -0,0 +1,522 @@
+from dataclasses import dataclass, field
+from einops import rearrange, repeat
+import math
+import torch
+from torch.amp.autocast_mode import autocast
+import torch.nn as nn
+from transformers.activations import ACT2FN
+from typing import cast
+
+# if flash_attn exists
+try:
+    from flash_attn.bert_padding import pad_input, unpad_input
+    from flash_attn.layers.rotary import RotaryEmbedding as FlashRotaryEmbedding
+    from flash_attn.modules.mha import FlashCrossAttention, FlashSelfAttention
+    from flash_attn.ops.fused_dense import FusedDense
+except ImportError:
+    print("flash_attn not found, using default implementations")
+    pad_input = unpad_input = FlashRotaryEmbedding = FlashCrossAttentio = FlashSelfAttention = FusedDense = None
+
+
+class RotaryEmbedding(nn.Module):
+    """Rotary positional embedding (RoPE) from Phi2.
+    See https://www.youtube.com/watch?v=C6rV8BsrrCc
+    """
+
+    def __init__(
+        self,
+        d_rotary: int,
+        rotary_base: float = 10000.0,
+        initial_cos_sin_cache_len: int = 2048,
+        device: torch.device | None = None,
+    ) -> None:
+        super().__init__()
+        self.d_rotary = d_rotary
+        self.rotary_base = rotary_base
+        self.device = device
+        self.dtype = torch.float32
+        self._update_cos_sin_cache(seqlen=initial_cos_sin_cache_len)
+
+    def _update_cos_sin_cache(self, seqlen: int) -> None:
+        # only call this function when seqlen is larger than _max_seqlen
+        self._max_seqlen = seqlen
+
+        # m * theta_i = m * base^(-2i/d) = m * (1 / base^(2i/d)), where i in [1, d/2]
+        m = torch.arange(
+            seqlen,
+            device=self.device,
+            dtype=self.dtype,
+        )
+        theta_i = 1.0 / (
+            self.rotary_base ** (
+                torch.arange(
+                    start=0,
+                    end=self.d_rotary,
+                    step=2,
+                    device=self.device,
+                    dtype=self.dtype,
+                ) / self.d_rotary
+            )
+        )
+        # torch.outer, since torch.einsum converts from fp32 to fp16 if used with torch.amp
+        # TODO: does this matter if I'm disabling torch.autocast?
+        m_theta_i = torch.outer(m, theta_i)
+        self._cos_cached = torch.cos(m_theta_i).to(self.dtype)
+        self._sin_cached = torch.sin(m_theta_i).to(self.dtype)
+
+        # TODO: scale_base caching is labelled as not yet done in Phi2
+        """
+        if scale_base is not None:
+            scale = (
+                torch.arange(
+                    start=0,
+                    end=self.d_rotary,
+                    step=2,
+                    device=self.device,
+                    dtype=torch.float32,
+                ) + 0.4 * self.d_rotary
+            ) / (1.4 * self.d_rotary)
+            power = (
+                torch.arange(seqlen, dtype=scale.dtype, device=scale.device) - seqlen // 2
+            ) / scale_base
+            scale = scale.to(device=power.device) ** rearrange(power, "s -> s 1")
+            self._cos_cached = (torch.cos(m_theta_i) * scale).to(dtype)
+            self._sin_cached = (torch.sin(m_theta_i) * scale).to(dtype)
+        """
+
+    def _apply_rotary_emb_qkv(
+        self,
+        x: torch.FloatTensor,  # dim: (batch_size, seqlen, Optional[n_qkv], n_heads, d_head)
+        cos: torch.FloatTensor,  # dim: (_max_seqlen, d_rotary)
+        sin: torch.FloatTensor,  # dim: (_max_seqlen, d_rotary)
+    ) -> torch.FloatTensor:
+        seqlen = x.shape[1]
+        x1, x2 = x.chunk(2, dim=-1)  # dim: (batch_size, seqlen, Optional[n_qkv], n_heads, d_head/2)
+        broadcast_rearrange = "s d -> s 1 d" if x1.ndim == 4 else "s d -> s 1 1 d"
+        c, s = rearrange(cos[:seqlen], broadcast_rearrange), rearrange(sin[:seqlen], broadcast_rearrange)
+        x1, x2, c, s = [t.to(dtype=torch.float32) for t in [x1, x2, c, s]]  # make sure rotary embedding is in float32
+        return cast(
+            torch.FloatTensor,
+            torch.cat([x1 * c - x2 * s, x1 * s + x2 * c], dim=-1).to(x.dtype)
+        )
+
+    def forward(
+        self,
+        x: torch.FloatTensor,  # dim: (batch_size, seqlen, Optional[n_qkv], n_heads, d_head)
+        seqlen_offset: int = 0,  # each sequence is shifted by this amount - used in inference with KV cache
+    ) -> torch.FloatTensor:
+        if (
+            not self._max_seqlen
+            or self._max_seqlen < x.shape[1] + seqlen_offset
+            or self._cos_cached.device != x.device
+            or self._cos_cached.dtype != x.dtype
+            or (self.training and self._cos_cached.is_inference())
+        ):
+            self._update_cos_sin_cache(seqlen=x.shape[1] + seqlen_offset)
+        return self._apply_rotary_emb_qkv(
+            x,
+            cast(torch.FloatTensor, self._cos_cached[seqlen_offset:]),
+            cast(torch.FloatTensor, self._sin_cached[seqlen_offset:]),
+        )
+
+
+class SelfAttention(nn.Module):
+    """Self-attention layer, taken from Phi2 model."""
+
+    def __init__(
+        self,
+        qk_scale: float | None = None,  # will use 1/sqrt(d) if set to None
+        attention_dropout: float = 0.0,
+    ) -> None:
+        super().__init__()
+        self.qk_scale = qk_scale
+        self.dropout = nn.Dropout(attention_dropout)
+
+    # autocast is manually disabled to avoid `torch.einsum` using float16, which might lead to overflow
+    @autocast("cpu", enabled=False)
+    @autocast("cuda", enabled=False)
+    def forward(
+        self,
+        qkv: torch.FloatTensor,  # dim: (batch_size, seqlen, 3, n_heads, d_head)
+        causal: bool = True,
+        key_padding_mask: torch.BoolTensor | None = None,
+    ) -> torch.FloatTensor:
+        batch_size, seqlen = qkv.shape[0], qkv.shape[1]
+        q, k, v = qkv.unbind(dim=2)
+        q = q.to(torch.float32)
+        k = k.to(torch.float32)
+        qk_scale = self.qk_scale or 1.0 / math.sqrt(q.shape[-1])
+
+        scores = torch.einsum("bthd,bshd->bhts", q, k * qk_scale)
+
+        if key_padding_mask:
+            padding_mask = torch.full((batch_size, seqlen), -10000.0, dtype=scores.dtype, device=scores.device)
+            padding_mask.masked_fill_(key_padding_mask, 0.0)
+            scores = scores + rearrange(padding_mask, "b s -> b 1 1 s")
+
+        if causal:
+            causal_mask = torch.triu(torch.full((seqlen, seqlen), -10000.0, device=scores.device), 1)
+            scores = scores + causal_mask.to(dtype=scores.dtype)
+
+        attention = torch.softmax(scores, dim=-1).to(v.dtype)
+        attention = self.dropout(attention)
+
+        output = torch.einsum("bhts,bshd->bthd", attention, v)  # dim: (batch_size, seqlen, n_heads, d_head)
+        return cast(torch.FloatTensor, output)
+
+
+class CrossAttention(nn.Module):
+    """Cross-attention layer, taken from Phi2 model."""
+
+    def __init__(
+        self,
+        qk_scale: float | None = None,  # will use 1/sqrt(d) if set to None
+        attention_dropout: float = 0.0,
+    ) -> None:
+        super().__init__()
+        self.qk_scale = qk_scale
+        self.dropout = nn.Dropout(attention_dropout)
+
+    # autocast is manually disabled to avoid `torch.einsum` using float16, which might lead to overflow
+    @autocast("cpu", enabled=False)
+    @autocast("cuda", enabled=False)
+    def forward(
+        self,
+        q: torch.FloatTensor,  # dim: (batch_size, seqlen_q, n_heads, d_head)
+        kv: torch.FloatTensor,  # dim: (batch_size, seqlen_kv, 2, n_heads, d_head)
+        causal: bool = True,
+        key_padding_mask: torch.BoolTensor | None = None,
+    ) -> torch.FloatTensor:
+        batch_size, seqlen_q = q.shape[0], q.shape[1]
+        seqlen_k = kv.shape[1]
+        if kv.shape[3] != q.shape[2]:  # repeat kv n_heads dim to match q n_heads
+            kv = repeat(kv, "... hkv d -> ... (hkv g) d", g=q.shape[2] // kv.shape[3])
+        k, v = kv.unbind(dim=2)
+        q = cast(torch.FloatTensor, q.to(torch.float32))
+        k = k.to(torch.float32)
+        qk_scale = self.qk_scale or 1.0 / math.sqrt(q.shape[-1])
+
+        scores = torch.einsum("bthd,bshd->bhts", q, k * qk_scale)
+
+        if key_padding_mask:
+            padding_mask = torch.full((batch_size, seqlen_k), -10000.0, dtype=scores.dtype, device=scores.device)
+            padding_mask.masked_fill_(key_padding_mask, 0.0)
+            scores = scores + rearrange(padding_mask, "b s -> b 1 1 s")
+
+        if causal:
+            rows = rearrange(torch.arange(seqlen_q, device=q.device, dtype=torch.long), "s -> s 1")
+            cols = torch.arange(seqlen_k, device=k.device, dtype=torch.long)
+            causal_mask = cols > rows + seqlen_k - seqlen_q
+            scores = scores.masked_fill(causal_mask, -10000.0)
+
+        attention = torch.softmax(scores, dim=-1).to(v.dtype)
+        attention = self.dropout(attention)
+
+        output = torch.einsum("bhts,bshd->bthd", attention, v)  # dim: (batch_size, seqlen_q, n_heads, d_head)
+        return cast(torch.FloatTensor, output)
+
+
+class MLP(nn.Module):
+    """Taken from Phi2 as well."""
+
+    def __init__(
+        self,
+        d_embedding: int,
+        act_fn: str = "gelu_new",
+    ) -> None:
+        super().__init__()
+        n_inner = 4 * d_embedding
+        self.fc1 = nn.Linear(d_embedding, n_inner)
+        self.act = ACT2FN[act_fn]
+        self.fc2 = nn.Linear(n_inner, d_embedding)
+
+    def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.fc2(x)
+        return x
+
+
+@dataclass
+class KVCache:
+    """Options for model to calculate and store context during inference."""
+    max_seqlen: int
+    max_batch_size: int
+    seqlen_offset: int
+    batch_size_offset: int
+    kv_block_map: dict[int, torch.Tensor] = field(default_factory=dict)
+    lengths_per_sample: torch.Tensor | None = None
+
+
+class MHA(nn.Module):
+    """Multi-head attention block."""
+
+    def __init__(
+        self,
+        d_embedding: int,
+        n_attn_heads: int,
+        block_n: int,
+        initial_cos_sin_cache_len: int,  # length of cache for rotary embedding
+        attn_pdrop: float,
+        use_flash_rotary: bool,  # use flash rotary embedding if possible
+        use_flash_attn: bool,  # use flash attention if possible
+        use_fused_dense: bool,  # use fused dense layer if possible
+        checkpointing: bool,  # torch.utils.checkpoint
+    ) -> None:
+        super().__init__()
+
+        # rotary embedding
+        rotary_cls = (
+            FlashRotaryEmbedding
+            if use_flash_rotary and FlashRotaryEmbedding is not None
+            else RotaryEmbedding
+        )
+        self.rotary_emb = rotary_cls(
+            d_rotary=math.ceil((d_embedding // n_attn_heads) / 2),  # d_rotary is half of d_head
+            initial_cos_sin_cache_len=initial_cos_sin_cache_len,
+        )
+
+        # self attention
+        self_attn_cls = (
+            FlashSelfAttention
+            if use_flash_attn and FlashSelfAttention is not None
+            else SelfAttention
+        )
+        self.inner_self_attn = self_attn_cls(attention_dropout=attn_pdrop)
+
+        # cross attention
+        cross_attn_cls = (
+            FlashCrossAttention
+            if use_flash_attn and FlashCrossAttention is not None
+            else CrossAttention
+        )
+        self.inner_cross_attn = cross_attn_cls(attention_dropout=attn_pdrop)
+
+        # MLP
+        self.n_attn_heads = n_attn_heads
+        self.d_head = d_embedding // n_attn_heads
+        linear_cls = (
+            FusedDense
+            if use_fused_dense and FusedDense is not None
+            else nn.Linear
+        )
+        self.Wqkv = linear_cls(
+            d_embedding,
+            self.d_head * (3 * self.n_attn_heads),  # calculating q, k, v for all heads in block simultaneously
+        )
+        self.fc_out = linear_cls(d_embedding, d_embedding)
+
+        # settings
+        self.using_flash_attn = self_attn_cls is FlashSelfAttention
+        self.block_n = block_n
+        self.checkpointing = checkpointing
+
+    def _forward_self_attn(
+        self,
+        qkv: torch.FloatTensor,  # dim: (batch_size, seqlen, 3, n_heads, d_head)
+        key_padding_mask: torch.BoolTensor | None,
+    ) -> torch.FloatTensor:
+        qkv = cast(
+            torch.FloatTensor,
+            torch.cat(
+                [
+                    self.rotary_emb(qkv[:, :, :2, :, :]),  # qk
+                    qkv[:, :, 2, :, :],  # v
+                ],
+                dim=2,
+            )
+        )
+
+        if self.using_flash_attn and unpad_input and pad_input:  # not touching flash attention code
+            batch_size, seqlen = qkv.shape[0], qkv.shape[1]
+            cu_seqlens, max_seqlen, indices = None, None, None
+
+            # unpad input and retrieve `cu_seqlens` and `max_seqlen` to be used by `flash-attn`
+            if key_padding_mask:
+                qkv, indices, cu_seqlens, max_seqlen = unpad_input(qkv, key_padding_mask)
+
+            if self.checkpointing:
+                attn_output = torch.utils.checkpoint.checkpoint(
+                    self.inner_self_attn, qkv, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen
+                )
+            else:
+                attn_output = self.inner_self_attn(qkv, cu_seqlens=cu_seqlens, max_seqlen=max_seqlen).to(qkv.device)
+
+            # repad output
+            if key_padding_mask:
+                return pad_input(attn_output, indices, batch_size, seqlen)
+            else:
+                return attn_output
+
+        if self.checkpointing:
+            return torch.utils.checkpoint.checkpoint(self.inner_self_attn, qkv, key_padding_mask=key_padding_mask)
+        else:
+            return self.inner_self_attn(qkv, key_padding_mask=key_padding_mask)
+
+    def _update_kv_cache(
+        self,
+        kv: torch.FloatTensor,  # dim: (batch_size, seqlen, 2, n_heads, d_head)
+        kv_cache: KVCache,
+        block_n: int,
+    ) -> None:
+        if block_n not in kv_cache.kv_block_map:
+            kv_cache.kv_block_map[block_n] = torch.empty(
+                kv_cache.max_batch_size,
+                kv_cache.max_seqlen,
+                2,
+                kv.shape[-2],  # n_heads
+                kv.shape[-1],  # d_head
+                dtype=kv.dtype,
+                device=kv.device,
+            )
+        kv_cache.kv_block_map[block_n][
+            kv_cache.batch_size_offset: kv_cache.batch_size_offset + kv.shape[0],
+            kv_cache.seqlen_offset: kv_cache.seqlen_offset + kv.shape[1],
+            ...
+        ] = kv
+
+    def _forward_cross_attn(
+        self,
+        qkv: torch.FloatTensor,  # dim: (batch_size, seqlen, 3, n_heads, d_head)
+        kv_cache: KVCache,
+        key_padding_mask: torch.BoolTensor | None,
+    ) -> torch.FloatTensor:
+        q = qkv[:, :, 0, :, :]
+        q = self.rotary_emb(
+            q,
+            seqlen_offset = 0 if kv_cache is None else kv_cache.seqlen_offset,
+        )
+        kv = cast(torch.FloatTensor, qkv[:, :, 1:, :, :])
+        self._update_kv_cache(kv, kv_cache, self.block_n)
+        causal = False  # turning off causal mask for cross attention
+
+        if self.using_flash_attn and unpad_input and pad_input:  # not touching flash attention code
+            batch_size, seqlen_q = q.shape[0], q.shape[1]
+            seqlen_k = kv.shape[1]
+            cu_seqlens_q, cu_seqlens_k, max_seqlen_q, max_seqlen_k, indices_q = (
+                None,
+                None,
+                None,
+                None,
+                None,
+            )
+
+            # unpad input and retrieve `cu_seqlens` and `max_seqlen` to be used by `flash-attn`
+            if key_padding_mask:
+                kv, _, cu_seqlens_k, max_seqlen_k = unpad_input(kv, key_padding_mask)
+
+                if seqlen_q == 1:
+                    key_padding_mask = cast(torch.BoolTensor, torch.ones(batch_size, 1, device=q.device))
+                elif seqlen_q != seqlen_k:
+                    key_padding_mask = cast(torch.BoolTensor, key_padding_mask[:, -seqlen_q:])
+
+                q, indices_q, cu_seqlens_q, max_seqlen_q = unpad_input(q, key_padding_mask)
+
+            if self.checkpointing:
+                attn_output = torch.utils.checkpoint.checkpoint(
+                    self.inner_cross_attn,
+                    q,
+                    kv,
+                    causal=causal,
+                    cu_seqlens=cu_seqlens_q,
+                    max_seqlen=max_seqlen_q,
+                    cu_seqlens_k=cu_seqlens_k,
+                    max_seqlen_k=max_seqlen_k,
+                )
+            else:
+                attn_output = self.inner_cross_attn(
+                    q,
+                    kv,
+                    causal=causal,
+                    cu_seqlens=cu_seqlens_q,
+                    max_seqlen=max_seqlen_q,
+                    cu_seqlens_k=cu_seqlens_k,
+                    max_seqlen_k=max_seqlen_k,
+                )
+
+            if key_padding_mask:
+                return pad_input(attn_output, indices_q, batch_size, max_seqlen_q)
+            else:
+                return attn_output
+
+        if self.checkpointing:
+            return torch.utils.checkpoint.checkpoint(
+                self.inner_cross_attn,
+                q,
+                kv,
+                key_padding_mask=key_padding_mask,
+                causal=causal,
+            )
+        else:
+            return self.inner_cross_attn(q, kv, key_padding_mask=key_padding_mask, causal=causal)
+
+    def forward(
+        self,
+        x: torch.FloatTensor,  # dim: (batch_size, seqlen, d_embedding)
+        kv_cache: KVCache | None = None,
+        key_padding_mask: torch.LongTensor | torch.BoolTensor | None = None,
+    ) -> tuple[torch.FloatTensor, torch.FloatTensor]:
+        if key_padding_mask is not None:
+            key_padding_mask = cast(torch.BoolTensor, key_padding_mask.bool())  # make sure it's bool and not int
+
+        qkv = self.Wqkv(x)  # dim: (batch_size, seqlen, 3*n_heads*d_head)
+        qkv = rearrange(qkv, "... (three h d) -> ... three h d", three=3, d=self.d_head)  # dim: (batch_size, seqlen, 3, n_heads, d_head)
+        if kv_cache is None:
+            attn_output = self._forward_self_attn(qkv, key_padding_mask)
+        else:
+            attn_output = self._forward_cross_attn(qkv, kv_cache, key_padding_mask)
+
+        output = rearrange(attn_output, "... h d -> ... (h d)")
+        output = self.fc_out(output)
+        return output
+
+
+class ParallelAttentionBlock(nn.Module):
+    """From Phi2. Calculates attention and MLP in parallel. See 'Simplifying Transformer Blocks', Fig. 1 'Parallel'."""
+
+    def __init__(
+        self,
+        resid_pdrop: float,  # a bit of a misnomer, right?
+        layer_norm_epsilon: float,
+        d_embedding: int,
+        n_attn_heads: int,
+        block_n: int,
+        initial_cos_sin_cache_len: int,  # length of cache for rotary embedding
+        attn_pdrop: float,
+        use_flash_rotary: bool = True,  # use flash rotary embedding if possible
+        use_flash_attn: bool = True,  # use flash attention if possible
+        use_fused_dense: bool = True,  # use fused dense layer if possible
+        checkpointing: bool = False,  # torch.utils.checkpoint
+    ) -> None:
+        super().__init__()
+        self.layer_norm = nn.LayerNorm(d_embedding, eps=layer_norm_epsilon)
+        self.block_n = block_n
+        self.multi_head_attention = MHA(
+            d_embedding=d_embedding,
+            n_attn_heads=n_attn_heads,
+            block_n=block_n,
+            initial_cos_sin_cache_len=initial_cos_sin_cache_len,
+            attn_pdrop=attn_pdrop,
+            use_flash_rotary=use_flash_rotary,
+            use_flash_attn=use_flash_attn,
+            use_fused_dense=use_fused_dense,
+            checkpointing=checkpointing,
+        )
+        self.mlp = MLP(d_embedding)
+        self.dropout = nn.Dropout(resid_pdrop)
+
+    def forward(
+        self,
+        x: torch.FloatTensor,  # dim: (batch_size, seq_len, d_embedding)
+        kv_cache: KVCache | None = None,
+        key_padding_mask: torch.BoolTensor | None = None,
+    ) -> torch.FloatTensor:
+        residual = x
+        x = self.layer_norm(x)  # each token (dim: d_embedding) is normalized individually
+        attn_outputs = self.multi_head_attention(
+            x,
+            kv_cache=kv_cache,
+            key_padding_mask=key_padding_mask,
+        )
+        mlp_outputs = self.mlp(x)
+        return self.dropout(attn_outputs + mlp_outputs) + residual

config.json

@@ -1,29 +1,32 @@
 {
-  "_name_or_path": "microsoft/phi-2",
-  "activation_function": "gelu_new",
-  "architectures": [
-    "PhiForCausalLM"
-  ],
-  "attn_pdrop": 0.0,
-  "auto_map": {
-    "AutoConfig": "configuration_phi.PhiConfig",
-    "AutoModelForCausalLM": "modeling_phi.PhiForCausalLM"
-  },
-  "embd_pdrop": 0.0,
-  "img_processor": null,
-  "initializer_range": 0.02,
-  "layer_norm_epsilon": 1e-05,
-  "model_type": "phi-msft",
-  "n_embd": 2560,
-  "n_head": 32,
-  "n_head_kv": null,
-  "n_inner": null,
-  "n_layer": 32,
-  "n_positions": 2048,
-  "resid_pdrop": 0.1,
-  "rotary_dim": 32,
-  "tie_word_embeddings": false,
-  "torch_dtype": "float16",
-  "transformers_version": "4.35.2",
-  "vocab_size": 51200
+    "_name_or_path": "BucketOfFish/simplified_phi2",
+    "architectures": [
+        "Phi2Model",
+        "Phi2ModelForCausalLM"
+    ],
+    "auto_map": {
+        "AutoConfig": "phi2_configuration.Phi2Config",
+        "AutoModel": "phi2_model.Phi2Model",
+        "AutoModelForCausalLM": "phi2_model.Phi2ModelForCausalLM"
+    },
+    "model_type": "phi2",
+    "torch_dtype": "float16",
+    "transformers_version": "4.29.0",
+
+    "vocab_size": 50304,
+    "vocab_chunk_for_gpu_efficiency": 64,
+    "initial_cos_sin_cache_len": 2048,
+    "d_embedding": 2560,
+    "n_blocks": 32,
+    "n_heads": 32,
+    "use_flash_attn": false,
+    "use_flash_rotary": false,
+    "use_fused_dense": false,
+    "attn_pdrop": 0.0,
+    "embd_pdrop": 0.0,
+    "resid_pdrop": 0.1,
+    "layer_norm_epsilon": 1e-05,
+    "weight_initialization_range": 0.02,
+    "tie_word_embeddings": false,
+    "checkpointing": false
 }

configuration_phi.py

@@ -1,56 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT license.
-
-import math
-from typing import Optional
-
-from transformers import PretrainedConfig
-
-
-class PhiConfig(PretrainedConfig):
-    """Phi configuration."""
-
-    model_type = "phi-msft"
-    attribute_map = {
-        "max_position_embeddings": "n_positions",
-        "hidden_size": "n_embd",
-        "num_attention_heads": "n_head",
-        "num_hidden_layers": "n_layer",
-    }
-
-    def __init__(
-        self,
-        vocab_size: int = 50304,
-        n_positions: int = 2048,
-        n_embd: int = 1024,
-        n_layer: int = 20,
-        n_inner: Optional[int] = None,
-        n_head: int = 16,
-        n_head_kv: Optional[int] = None,
-        rotary_dim: Optional[int] = 32,
-        activation_function: Optional[str] = "gelu_new",
-        attn_pdrop: float = 0.0,
-        embd_pdrop: float = 0.0,
-        resid_pdrop: float = 0.0,
-        layer_norm_epsilon: float = 1e-5,
-        initializer_range: float = 0.02,
-        tie_word_embeddings: bool = False,
-        pad_vocab_size_multiple: int = 64,
-        **kwargs
-    ) -> None:
-        self.vocab_size = int(math.ceil(vocab_size / pad_vocab_size_multiple) * pad_vocab_size_multiple)
-        self.n_positions = n_positions
-        self.n_embd = n_embd
-        self.n_layer = n_layer
-        self.n_inner = n_inner
-        self.n_head = n_head
-        self.n_head_kv = n_head_kv
-        self.rotary_dim = min(rotary_dim, n_embd // n_head)
-        self.activation_function = activation_function
-        self.attn_pdrop = attn_pdrop
-        self.embd_pdrop = embd_pdrop
-        self.resid_pdrop = resid_pdrop
-        self.layer_norm_epsilon = layer_norm_epsilon
-        self.initializer_range = initializer_range
-
-        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)

modeling_phi.py

@@ -1,766 +0,0 @@
-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT license.
-#
-# Copyright (c) 2022, Tri Dao, [email protected].
-# Licensed under the BSD 3-Clause License.
-
-from __future__ import annotations
-
-import math
-from dataclasses import dataclass, field
-from typing import Any, Dict, Optional, Tuple, Union
-
-import torch
-import torch.nn as nn
-from einops import rearrange, repeat
-from transformers import PretrainedConfig, PreTrainedModel
-from transformers.activations import ACT2FN
-from transformers.modeling_outputs import CausalLMOutputWithPast
-
-from .configuration_phi import PhiConfig
-
-
-@dataclass
-class InferenceParams:
-    """Inference parameters passed to model to efficiently calculate
-    and store context during inference.
-
-    Reference:
-        https://github.com/Dao-AILab/flash-attention/blob/main/flash_attn/utils/generation.py.
-
-    Args:
-        max_seqlen: Maximum sequence length.
-        max_batch_size: Maximum batch size.
-        seqlen_offset: Sequence length offset.
-        batch_size_offset: Batch size offset.
-        key_value_memory_dict: Key value memory dictionary.
-        lengths_per_sample: Lengths per sample.
-
-    """
-
-    max_seqlen: int = field(metadata={"help": "Maximum sequence length."})
-
-    max_batch_size: int = field(metadata={"help": "Maximum batch size."})
-
-    seqlen_offset: int = field(default=0, metadata={"help": "Sequence length offset."})
-
-    batch_size_offset: int = field(default=0, metadata={"help": "Batch size offset."})
-
-    key_value_memory_dict: Dict[str, Any] = field(
-        default_factory=dict, metadata={"help": "Key value memory dictionary."}
-    )
-
-    lengths_per_sample: torch.Tensor = field(default=None, metadata={"help": "Lengths per sample."})
-
-
-class Embedding(nn.Module):
-    """Token embedding with dropout."""
-
-    def __init__(self, config: PretrainedConfig) -> None:
-        super().__init__()
-
-        self.wte = nn.Embedding(config.vocab_size, config.n_embd)
-        self.drop = nn.Dropout(config.embd_pdrop)
-
-    def forward(self, input_ids: torch.LongTensor) -> torch.FloatTensor:
-        input_shape = input_ids.size()
-        input_ids = input_ids.view(-1, input_shape[-1])
-
-        hidden_states = self.wte(input_ids)
-        hidden_states = self.drop(hidden_states)
-
-        return hidden_states
-
-
-class RotaryEmbedding(nn.Module):
-    """Rotary positional embedding (RoPE) from Phi2.
-    See https://www.youtube.com/watch?v=C6rV8BsrrCc
-    """
-
-    def __init__(
-        self,
-        d_rotary: int,
-        rotary_base: float = 10000.0,
-        initial_cos_sin_cache_len: int = 2048,
-        device: torch.device | None = None,
-    ) -> None:
-        super().__init__()
-        self.d_rotary = d_rotary
-        self.rotary_base = rotary_base
-        self.device = device
-        self.dtype = torch.float32
-        self._update_cos_sin_cache(seqlen=initial_cos_sin_cache_len)
-
-    def _update_cos_sin_cache(self, seqlen: int) -> None:
-        # only call this function when seqlen is larger than _max_seqlen
-        self._max_seqlen = seqlen
-
-        # m * theta_i = m * base^(-2i/d) = m * (1 / base^(2i/d)), where i in [1, d/2]
-        m = torch.arange(
-            seqlen,
-            device=self.device,
-            dtype=self.dtype,
-        )
-        theta_i = 1.0 / (
-            self.rotary_base ** (
-                torch.arange(
-                    start=0,
-                    end=self.d_rotary,
-                    step=2,
-                    device=self.device,
-                    dtype=self.dtype,
-                ) / self.d_rotary
-            )
-        )
-        # torch.outer, since torch.einsum converts from fp32 to fp16 if used with torch.amp
-        # TODO: does this matter if I'm disabling torch.autocast?
-        m_theta_i = torch.outer(m, theta_i)
-        self._cos_cached = torch.cos(m_theta_i).to(self.dtype)
-        self._sin_cached = torch.sin(m_theta_i).to(self.dtype)
-
-        # TODO: scale_base caching is labelled as not yet done in Phi2
-        """
-        if scale_base is not None:
-            scale = (
-                torch.arange(
-                    start=0,
-                    end=self.d_rotary,
-                    step=2,
-                    device=self.device,
-                    dtype=torch.float32,
-                ) + 0.4 * self.d_rotary
-            ) / (1.4 * self.d_rotary)
-            power = (
-                torch.arange(seqlen, dtype=scale.dtype, device=scale.device) - seqlen // 2
-            ) / scale_base
-            scale = scale.to(device=power.device) ** rearrange(power, "s -> s 1")
-            self._cos_cached = (torch.cos(m_theta_i) * scale).to(dtype)
-            self._sin_cached = (torch.sin(m_theta_i) * scale).to(dtype)
-        """
-
-    def _apply_rotary_emb_qkv(
-        self,
-        x: torch.FloatTensor,  # dim: (batch_size, seqlen, Optional[n_qkv], n_heads, d_head)
-        cos: torch.FloatTensor,  # dim: (_max_seqlen, d_rotary)
-        sin: torch.FloatTensor,  # dim: (_max_seqlen, d_rotary)
-    ) -> torch.FloatTensor:
-        seqlen = x.shape[1]
-        x1, x2 = x.chunk(2, dim=-1)  # dim: (batch_size, seqlen, Optional[n_qkv], n_heads, d_head/2)
-        broadcast_rearrange = "s d -> s 1 d" if x1.ndim == 4 else "s d -> s 1 1 d"
-        c, s = rearrange(cos[:seqlen], broadcast_rearrange), rearrange(sin[:seqlen], broadcast_rearrange)
-        x1, x2, c, s = [t.to(dtype=torch.float32) for t in [x1, x2, c, s]]  # make sure rotary embedding is in float32
-        return cast(
-            torch.FloatTensor,
-            torch.cat([x1 * c - x2 * s, x1 * s + x2 * c], dim=-1).to(x.dtype)
-        )
-
-    def forward(
-        self,
-        x: torch.FloatTensor,  # dim: (batch_size, seqlen, Optional[n_qkv], n_heads, d_head)
-        seqlen_offset: int = 0,  # each sequence is shifted by this amount - used in inference with KV cache
-    ) -> torch.FloatTensor:
-        if (
-            not self._max_seqlen
-            or self._max_seqlen < x.shape[1] + seqlen_offset
-            or self._cos_cached.device != x.device
-            or self._cos_cached.dtype != x.dtype
-            or (self.training and self._cos_cached.is_inference())
-        ):
-            self._update_cos_sin_cache(seqlen=x.shape[1] + seqlen_offset)
-        return self._apply_rotary_emb_qkv(
-            x,
-            cast(torch.FloatTensor, self._cos_cached[seqlen_offset:]),
-            cast(torch.FloatTensor, self._sin_cached[seqlen_offset:]),
-        )
-
-
-class MLP(nn.Module):
-    """Multi-Layer Perceptron.
-
-    Reference:
-        Attention Is All You Need.
-        https://arxiv.org/pdf/1706.03762.pdf.
-
-    """
-
-    def __init__(
-        self,
-        config: PretrainedConfig,
-        n_inner: Optional[int] = None,
-        act_fn: Optional[str] = None,
-    ) -> None:
-        super().__init__()
-
-        act_fn = config.activation_function if act_fn is None else act_fn
-
-        n_inner = getattr(config, "n_inner", None) if n_inner is None else n_inner
-        n_inner = n_inner if n_inner is not None else 4 * config.n_embd
-
-        self.fc1 = nn.Linear(config.n_embd, n_inner)
-        self.fc2 = nn.Linear(n_inner, config.n_embd)
-        self.act = ACT2FN[act_fn]
-
-    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
-        hidden_states = self.fc1(hidden_states)
-        hidden_states = self.act(hidden_states)
-        hidden_states = self.fc2(hidden_states)
-
-        return hidden_states
-
-
-class SelfAttention(nn.Module):
-    """Self-attention layer (compatible with PyTorch).
-
-    Reference:
-        https://github.com/Dao-AILab/flash-attention/blob/main/flash_attn/modules/mha.py.
-
-    """
-
-    def __init__(
-        self,
-        causal: bool = True,
-        softmax_scale: Optional[float] = None,
-        attention_dropout: float = 0.0,
-    ) -> None:
-        super().__init__()
-
-        self.causal = causal
-        self.softmax_scale = softmax_scale
-        self.drop = nn.Dropout(attention_dropout)
-
-    @torch.autocast("cpu", enabled=False)
-    @torch.autocast("cuda", enabled=False)
-    def forward(
-        self,
-        qkv: torch.FloatTensor,
-        causal: bool = None,
-        key_padding_mask: Optional[torch.BoolTensor] = None,
-        **kwargs,
-    ) -> torch.FloatTensor:
-        batch_size, seqlen = qkv.shape[0], qkv.shape[1]
-        q, k, v = qkv.unbind(dim=2)
-
-        q = q.to(torch.float32)
-        k = k.to(torch.float32)
-
-        causal = self.causal if causal is None else causal
-        softmax_scale = self.softmax_scale or 1.0 / math.sqrt(q.shape[-1])
-
-        # Autocast is manually disabled to avoid `torch.einsum` performing the operation
-        # using float16, which might lead to overflow
-        scores = torch.einsum("bthd,bshd->bhts", q, k * softmax_scale)
-
-        if key_padding_mask is not None:
-            padding_mask = torch.full((batch_size, seqlen), -10000.0, dtype=scores.dtype, device=scores.device)
-            padding_mask.masked_fill_(key_padding_mask, 0.0)
-
-            scores = scores + rearrange(padding_mask, "b s -> b 1 1 s")
-
-        if causal:
-            causal_mask = torch.triu(torch.full((seqlen, seqlen), -10000.0, device=scores.device), 1)
-            scores = scores + causal_mask.to(dtype=scores.dtype)
-
-        attention = torch.softmax(scores, dim=-1).to(v.dtype)
-        attention = self.drop(attention)
-
-        output = torch.einsum("bhts,bshd->bthd", attention, v)
-
-        return output
-
-
-class CrossAttention(nn.Module):
-    """Cross-attention layer (compatible with PyTorch).
-
-    Reference:
-        https://github.com/Dao-AILab/flash-attention/blob/main/flash_attn/modules/mha.py.
-
-    """
-
-    def __init__(
-        self,
-        causal: bool = True,
-        softmax_scale: Optional[float] = None,
-        attention_dropout: float = 0.0,
-    ) -> None:
-        super().__init__()
-
-        self.causal = causal
-        self.softmax_scale = softmax_scale
-        self.drop = nn.Dropout(attention_dropout)
-
-    @torch.autocast("cpu", enabled=False)
-    @torch.autocast("cuda", enabled=False)
-    def forward(
-        self,
-        q: torch.FloatTensor,
-        kv: torch.FloatTensor,
-        causal: bool = None,
-        key_padding_mask: Optional[torch.BoolTensor] = None,
-        **kwargs,
-    ) -> torch.FloatTensor:
-        batch_size, seqlen_q = q.shape[0], q.shape[1]
-        seqlen_k = kv.shape[1]
-
-        if kv.shape[3] != q.shape[2]:
-            kv = repeat(kv, "... hkv d -> ... (hkv g) d", g=q.shape[2] // kv.shape[3])
-        k, v = kv.unbind(dim=2)
-
-        q = q.to(torch.float32)
-        k = k.to(torch.float32)
-
-        causal = self.causal if causal is None else causal
-        softmax_scale = self.softmax_scale or 1.0 / math.sqrt(q.shape[-1])
-
-        # Autocast is manually disabled to avoid `torch.einsum` performing the operation
-        # using float16, which might lead to overflow
-        scores = torch.einsum("bthd,bshd->bhts", q, k * softmax_scale)
-
-        if key_padding_mask is not None:
-            padding_mask = torch.full(
-                (batch_size, seqlen_k),
-                -10000.0,
-                dtype=scores.dtype,
-                device=scores.device,
-            )
-            padding_mask.masked_fill_(key_padding_mask, 0.0)
-
-            scores = scores + rearrange(padding_mask, "b s -> b 1 1 s")
-
-        if causal:
-            rows = rearrange(torch.arange(seqlen_q, device=q.device, dtype=torch.long), "s -> s 1")
-            cols = torch.arange(seqlen_k, device=k.device, dtype=torch.long)
-            causal_mask = cols > rows + seqlen_k - seqlen_q
-
-            scores = scores.masked_fill(causal_mask, -10000.0)
-
-        attention = torch.softmax(scores, dim=-1).to(v.dtype)
-        attention = self.drop(attention)
-
-        output = torch.einsum("bhts,bshd->bthd", attention, v)
-
-        return output
-
-
-def _find_mha_dims(
-    config: PretrainedConfig,
-    n_head: Optional[int] = None,
-    n_head_kv: Optional[int] = None,
-    head_dim: Optional[int] = None,
-) -> Tuple[int, int]:
-    if n_head is None and head_dim is None:
-        head_dim = config.n_embd // config.n_head
-        n_head = config.n_head
-    elif n_head is None or head_dim is None:
-        raise ValueError("`n_head` and `head_dim` must be both specified or `None`.")
-
-    if n_head_kv is None:
-        n_head_kv = getattr(config, "n_head_kv", None) or n_head
-
-    return n_head, n_head_kv, head_dim
-
-
-def _update_kv_cache(kv: torch.FloatTensor, inference_params: InferenceParams, layer_idx: int) -> torch.FloatTensor:
-    num_heads, head_dim = kv.shape[-2:]
-
-    if layer_idx not in inference_params.key_value_memory_dict:
-        inference_params.key_value_memory_dict[layer_idx] = torch.empty(
-            inference_params.max_batch_size,
-            inference_params.max_seqlen,
-            2,
-            num_heads,
-            head_dim,
-            dtype=kv.dtype,
-            device=kv.device,
-        )
-
-    batch_start = inference_params.batch_size_offset
-    batch_end = batch_start + kv.shape[0]
-
-    sequence_start = inference_params.seqlen_offset
-    sequence_end = sequence_start + kv.shape[1]
-
-    # When the current sequence length is equal to or larger than the maximum sequence length,
-    # we need to concatenate the current `kv` with the cached `kv` to expand its length
-    if sequence_end >= inference_params.max_seqlen:
-        inference_params.key_value_memory_dict[layer_idx] = torch.concatenate((inference_params.key_value_memory_dict[layer_idx], kv), dim=1)
-
-    inference_params.key_value_memory_dict[layer_idx][batch_start:batch_end, sequence_start:sequence_end, ...] = kv
-    kv = inference_params.key_value_memory_dict[layer_idx][batch_start:batch_end, :sequence_end, ...]
-        
-    return kv
-
-
-class MHA(nn.Module):
-    """Multi-head attention layer."""
-
-    def __init__(
-        self,
-        config: PretrainedConfig,
-        dtype: Optional[torch.dtype] = None,
-        device: Optional[str] = None,
-        rotary_dim: Optional[int] = None,
-        rotary_base: float = 10000.0,
-        rotary_scale_base: Optional[float] = None,
-        n_head: Optional[int] = None,
-        n_head_kv: Optional[int] = None,
-        head_dim: Optional[int] = None,
-        bias: bool = True,
-        causal: bool = True,
-        softmax_scale: Optional[float] = None,
-        layer_idx: Optional[int] = None,
-        return_residual: bool = False,
-        checkpointing: bool = False,
-    ) -> None:
-        super().__init__()
-
-        # Rotary embedding
-        self.rotary_dim = rotary_dim if rotary_dim is not None else getattr(config, "rotary_dim", 0)
-        if self.rotary_dim > 0:
-            self.rotary_emb = RotaryEmbedding(
-                d_rotary=self.rotary_dim,
-                # d_rotary=math.ceil((rotary_dim // n_head) / 2),  # d_rotary is half of d_head
-                initial_cos_sin_cache_len=config.n_positions,
-            )
-
-        # MLP
-        self.n_head, self.n_head_kv, self.head_dim = _find_mha_dims(
-            config, n_head=n_head, n_head_kv=n_head_kv, head_dim=head_dim
-        )
-        op_size = self.head_dim * (self.n_head + 2 * self.n_head_kv)
-        hidden_size = config.n_embd
-
-        linear_cls = nn.Linear
-        if linear_cls is None:
-            linear_cls = nn.Linear
-
-        self.Wqkv = linear_cls(hidden_size, op_size, bias=bias, device=device, dtype=dtype)
-        self.out_proj = linear_cls(hidden_size, hidden_size, bias=bias, device=device, dtype=dtype)
-
-        # Attention
-        attn_cls = SelfAttention
-        if attn_cls is None:
-            attn_cls = SelfAttention
-
-        cross_attn_cls = CrossAttention
-        if cross_attn_cls is None:
-            cross_attn_cls = CrossAttention
-
-        self.inner_attn = attn_cls(
-            causal=causal,
-            softmax_scale=softmax_scale,
-            attention_dropout=config.attn_pdrop,
-        )
-        self.inner_cross_attn = cross_attn_cls(
-            causal=causal,
-            softmax_scale=softmax_scale,
-            attention_dropout=config.attn_pdrop,
-        )
-
-        self.layer_idx = layer_idx
-        self.return_residual = return_residual
-        self.checkpointing = checkpointing
-
-    def _forward_self_attn(
-        self, x: torch.FloatTensor, key_padding_mask: Optional[torch.BoolTensor]
-    ) -> torch.FloatTensor:
-        qkv = self.Wqkv(x)
-        qkv = rearrange(qkv, "... (three h d) -> ... three h d", three=3, d=self.head_dim)
-
-        if self.rotary_dim > 0:
-            qkv = self.rotary_emb(qkv)
-
-        if self.checkpointing:
-            return torch.utils.checkpoint.checkpoint(self.inner_attn, qkv, key_padding_mask=key_padding_mask)
-
-        return self.inner_attn(qkv, key_padding_mask=key_padding_mask)
-
-    def _forward_cross_attn(
-        self,
-        x: torch.FloatTensor,
-        past_key_values: Optional[InferenceParams],
-        key_padding_mask: Optional[torch.BoolTensor],
-    ) -> torch.FloatTensor:
-        batch_size = x.shape[0]
-
-        qkv = self.Wqkv(x)
-
-        q = qkv[..., : self.n_head * self.head_dim]
-        q = rearrange(q, "... (h d) -> ... h d", d=self.head_dim)
-
-        kv = qkv[..., self.n_head * self.head_dim :]
-        kv = rearrange(kv, "... (two hkv d) -> ... two hkv d", two=2, d=self.head_dim)
-
-        seqlen_offset = past_key_values.seqlen_offset if past_key_values is not None else 0
-        causal = None if seqlen_offset == 0 else False
-        if self.rotary_dim > 0:
-            q, kv = self.rotary_emb(q, kv=kv, seqlen_offset=seqlen_offset)
-
-        if past_key_values is not None:
-            kv = _update_kv_cache(kv, past_key_values, self.layer_idx)
-
-        if self.checkpointing:
-            return torch.utils.checkpoint.checkpoint(
-                self.inner_cross_attn,
-                q,
-                kv,
-                key_padding_mask=key_padding_mask,
-                causal=causal,
-            )
-
-        return self.inner_cross_attn(q, kv, key_padding_mask=key_padding_mask, causal=causal)
-
-    def forward(
-        self,
-        x: torch.FloatTensor,
-        past_key_values: Optional[InferenceParams] = None,
-        attention_mask: Optional[Union[torch.LongTensor, torch.BoolTensor]] = None,
-        **kwargs,
-    ) -> Tuple[torch.FloatTensor, torch.FloatTensor]:
-        if attention_mask is not None:
-            attention_mask = attention_mask.bool()
-        else:
-            attention_mask = None
-
-        # MHA
-        if self.n_head == self.n_head_kv:
-            if past_key_values is None:
-                # If `past_key_values` are not supplied, we run self-attention
-                attn_output = self._forward_self_attn(x, attention_mask)
-            else:
-                # If `past_key_values` are supplied, it means that we might have cached values and
-                # could take advantage of cross-attention
-                attn_output = self._forward_cross_attn(x, past_key_values, attention_mask)
-        # MQA / GQA
-        else:
-            # Regardless of `past_key_values` being supplied or not, it always use cross-attention
-            # because `q` and `kv` lengths might be different
-            attn_output = self._forward_cross_attn(x, past_key_values, attention_mask)
-
-        output = rearrange(attn_output, "... h d -> ... (h d)")
-        output = self.out_proj(output)
-
-        return output if not self.return_residual else (output, x)
-
-
-class ParallelBlock(nn.Module):
-    """Parallel block.
-
-    This block applies parallel mixer and MLP layers to the input (used in GPT-J and CodeGen).
-
-    """
-
-    def __init__(
-        self,
-        config: PretrainedConfig,
-        block_idx: Optional[int] = None,
-    ) -> None:
-        super().__init__()
-
-        self.ln = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
-        self.resid_dropout = nn.Dropout(config.resid_pdrop)
-        self.block_idx = block_idx
-
-        self.mixer = MHA(config, layer_idx=block_idx)
-        self.mlp = MLP(config)
-
-    def forward(
-        self,
-        hidden_states: torch.FloatTensor,
-        past_key_values: Optional[Union[torch.FloatTensor, InferenceParams]] = None,
-        attention_mask: Optional[torch.BoolTensor] = None,
-        **kwargs,
-    ) -> torch.FloatTensor:
-        residual = hidden_states
-        hidden_states = self.ln(hidden_states)
-
-        attn_outputs = self.mixer(
-            hidden_states,
-            past_key_values=past_key_values,
-            attention_mask=attention_mask,
-        )
-        if isinstance(attn_outputs, tuple):
-            attn_outputs = attn_outputs[0]
-
-        attn_outputs = self.resid_dropout(attn_outputs)
-        feed_forward_hidden_states = self.resid_dropout(self.mlp(hidden_states))
-
-        hidden_states = attn_outputs + feed_forward_hidden_states + residual
-
-        return hidden_states
-
-
-class CausalLMHead(nn.Module):
-    """Causal Language Modeling head.
-
-    Reference:
-        Improving Language Understanding by Generative Pre-Training.
-        https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf.
-
-    """
-
-    def __init__(self, config: PretrainedConfig) -> None:
-        super().__init__()
-
-        self.ln = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
-        self.linear = nn.Linear(config.n_embd, config.vocab_size)
-
-    def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
-        hidden_states = self.ln(hidden_states)
-        logits = self.linear(hidden_states).to(torch.float32)
-
-        return logits
-
-
-class CausalLMLoss(nn.Module):
-    """Causal Language Modeling loss.
-
-    Reference:
-        Improving Language Understanding by Generative Pre-Training.
-        https://cdn.openai.com/research-covers/language-unsupervised/language_understanding_paper.pdf.
-
-    """
-
-    def __init__(self, shift_labels: bool = True) -> None:
-        super().__init__()
-
-        self.shift_labels = shift_labels
-        self.loss_fct = nn.CrossEntropyLoss()
-
-    def forward(self, logits: torch.FloatTensor, labels: torch.LongTensor) -> torch.FloatTensor:
-        if self.shift_labels:
-            logits = logits[..., :-1, :].contiguous()
-            labels = labels[..., 1:].contiguous()
-
-        loss = self.loss_fct(logits.view(-1, logits.size(-1)), labels.view(-1))
-
-        return loss
-
-
-class PhiPreTrainedModel(PreTrainedModel):
-    """Phi pre-trained model."""
-
-    config_class = PhiConfig
-    base_model_prefix = "transformer"
-    supports_gradient_checkpointing = False
-    _no_split_modules = ["ParallelBlock"]
-
-    def __init__(self, *inputs, **kwargs) -> None:
-        super().__init__(*inputs, **kwargs)
-
-    def _init_weights(self, module: nn.Module) -> None:
-        if isinstance(module, (nn.Linear,)):
-            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
-            if module.bias is not None:
-                module.bias.data.zero_()
-        elif isinstance(module, nn.Embedding):
-            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
-            if module.padding_idx is not None:
-                module.weight.data[module.padding_idx].zero_()
-        elif isinstance(module, nn.LayerNorm):
-            if module.bias is not None:
-                module.bias.data.zero_()
-            module.weight.data.fill_(1.0)
-
-    def prepare_inputs_for_generation(
-        self,
-        input_ids: torch.LongTensor,
-        past_key_values: Optional[Union[torch.FloatTensor, InferenceParams]] = None,
-        attention_mask: Optional[Union[torch.LongTensor, torch.BoolTensor]] = None,
-        **kwargs,
-    ) -> Dict[str, Any]:
-        if past_key_values is None or not (isinstance(past_key_values, InferenceParams)):
-            past_key_values = InferenceParams(
-                max_seqlen=self.config.n_positions,
-                max_batch_size=input_ids.shape[0],
-                seqlen_offset=0,
-                batch_size_offset=0,
-                key_value_memory_dict={},
-                lengths_per_sample=None,
-            )
-        else:
-            # Assume that `past_key_values` has cached all tokens up to the last token in `input_ids`
-            past_key_values.seqlen_offset = input_ids.shape[1] - 1
-            input_ids = input_ids[:, -1].unsqueeze(-1)
-
-        return {
-            "input_ids": input_ids,
-            "past_key_values": past_key_values,
-            "attention_mask": attention_mask,
-        }
-
-
-class PhiModel(PhiPreTrainedModel):
-    """Phi model."""
-
-    _keys_to_ignore_on_load_missing = [""]
-    _keys_to_ignore_on_load_unexpected = [r"h\.\d+\.mlp.(fc_in|fc_out)\.(weight|bias)"]
-
-    def __init__(self, config: PhiConfig) -> None:
-        super().__init__(config)
-
-        self.embd = Embedding(config)
-        self.h = nn.ModuleList([ParallelBlock(config, block_idx=i) for i in range(config.n_layer)])
-        self.gradient_checkpointing = False
-        self.post_init()
-
-    def get_input_embeddings(self) -> nn.Embedding:
-        return self.embd.wte
-
-    def set_input_embeddings(self, new_embeddings: nn.Embedding) -> None:
-        self.embd.wte = new_embeddings
-
-    def forward(
-        self,
-        input_ids: torch.LongTensor,
-        past_key_values: Optional[Union[torch.FloatTensor, InferenceParams]] = None,
-        attention_mask: Optional[torch.BoolTensor] = None,
-    ) -> torch.FloatTensor:
-        hidden_states = self.embd(input_ids)
-
-        for layer in self.h:
-            hidden_states = layer(
-                hidden_states,
-                past_key_values=past_key_values,
-                attention_mask=attention_mask,
-            )
-
-        return hidden_states
-
-
-class PhiForCausalLM(PhiPreTrainedModel):
-    """Phi for Causal Language Modeling."""
-
-    _keys_to_ignore_on_load_missing = [""]
-    _keys_to_ignore_on_load_unexpected = [r"transformer\.h\.\d+\.mlp.(fc_in|fc_out)\.(weight|bias)"]
-
-    def __init__(self, config: PhiConfig) -> None:
-        super().__init__(config)
-
-        self.transformer = PhiModel(config)
-        self.lm_head = CausalLMHead(config)
-        self.loss = CausalLMLoss()
-
-        self.post_init()
-
-    def get_output_embeddings(self) -> nn.Linear:
-        return self.lm_head.linear
-
-    def set_output_embeddings(self, new_embeddings: nn.Linear) -> None:
-        self.lm_head.linear = new_embeddings
-
-    def forward(
-        self,
-        input_ids: torch.LongTensor,
-        past_key_values: Optional[Union[torch.FloatTensor, InferenceParams]] = None,
-        attention_mask: Optional[torch.BoolTensor] = None,
-        labels: Optional[torch.LongTensor] = None,
-        **kwargs,
-    ) -> CausalLMOutputWithPast:
-        hidden_states = self.transformer(input_ids, past_key_values=past_key_values, attention_mask=attention_mask)
-        lm_logits = self.lm_head(hidden_states)
-
-        loss = None
-        if labels is not None:
-            loss = self.loss(lm_logits, labels)
-
-        return CausalLMOutputWithPast(loss=loss, logits=lm_logits, past_key_values=past_key_values)

phi2_configuration.py

@@ -0,0 +1,60 @@
+import math
+from transformers import PretrainedConfig
+
+
+class Phi2Config(PretrainedConfig):
+    model_type = "phi2"  # not necessary unless you want to register model with auto classes
+    attribute_map = {
+        "max_position_embeddings": "initial_cos_sin_cache_len",
+        "hidden_size": "d_embedding",
+        "num_attention_heads": "n_attn_heads",
+        "num_hidden_layers": "n_blocks",
+    }
+
+    def __init__(
+        self,
+        vocab_size: int = 50295,  # this includes the extra tokens included by Phi2 in tokenizer_config.json
+        vocab_chunk_for_gpu_efficiency: int = 64,
+        initial_cos_sin_cache_len: int = 2048,
+        d_embedding: int = 1024,  # 2560?
+        n_blocks: int = 20,  # 32?
+        n_attn_heads: int = 16,  # 32?
+        use_flash_attn: bool = False,
+        use_flash_rotary: bool = False,
+        use_fused_dense: bool = False,
+        attn_pdrop: float = 0.0,
+        embd_pdrop: float = 0.0,
+        resid_pdrop: float = 0.0,
+        layer_norm_epsilon: float = 1e-5,
+        weight_initialization_range: float = 0.02,
+        tie_word_embeddings: bool = False,  # whether embedding weights are shared between the encoder and decoder
+        checkpointing: bool = False,  # whether to use gradient checkpointing to reduce memory usage (I think)
+        **kwargs
+    ) -> None:
+        self.vocab_size = (
+            math.ceil(
+                vocab_size / vocab_chunk_for_gpu_efficiency
+            ) * vocab_chunk_for_gpu_efficiency
+        )
+        self.initial_cos_sin_cache_len = initial_cos_sin_cache_len
+        self.d_embedding = d_embedding
+        self.n_blocks = n_blocks
+        self.n_attn_heads = n_attn_heads
+        self.use_flash_attn = use_flash_attn
+        self.use_flash_rotary = use_flash_rotary
+        self.use_fused_dense = use_fused_dense
+        self.attn_pdrop = attn_pdrop
+        self.embd_pdrop = embd_pdrop
+        self.resid_pdrop = resid_pdrop
+        self.layer_norm_epsilon = layer_norm_epsilon
+        self.weight_initialization_range = weight_initialization_range
+        self.checkpointing = checkpointing
+
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+
+
+if __name__ == "__main__":
+    phi2_config = Phi2Config()
+    # phi2_config.save_pretrained("phi2_config")
+    # phi2_config = Phi2Config.from_pretrained("phi2_config")
+    # phi2_config.push_to_hub("phi2_config")

phi2_model.py

@@ -0,0 +1,166 @@
+import torch
+import torch.nn as nn
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from typing import Any, cast
+
+from .attention import ParallelAttentionBlock, KVCache
+from .phi2_configuration import Phi2Config
+
+
+class Phi2PreTrainedModel(PreTrainedModel):
+    config_class = Phi2Config  # not necessary unless you want to register model with auto classes
+    supports_gradient_checkpointing = False
+    # _no_split_modules = ["ParallelAttentionBlock"]
+
+    # weight loading
+    # base_model_prefix = "transformer"
+    # _keys_to_ignore_on_load_missing = [""]
+    # _keys_to_ignore_on_load_unexpected = [r"h\.\d+\.mlp.(fc_in|fc_out)\.(weight|bias)"]
+
+    def __init__(self, config: Phi2Config):
+        super().__init__(config)
+        self.config = config
+
+    def _init_weights(self, module: nn.Module) -> None:
+        # initialize weights - will get overwritten by saved weights in from_pretrained() if they exist
+        if isinstance(module, (nn.Linear,)):
+            module.weight.data.normal_(mean=0.0, std=self.config.weight_initialization_range)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=self.config.weight_initialization_range)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+        elif isinstance(module, nn.LayerNorm):
+            if module.bias is not None:
+                module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids: torch.LongTensor,  # dim: (batch_size, seq_len)
+        kv_cache: KVCache | None = None,
+        key_padding_mask: torch.LongTensor | torch.BoolTensor | None = None,
+    ) -> dict[str, Any]:
+        if not kv_cache:
+            kv_cache = KVCache(
+                max_seqlen=self.config.initial_cos_sin_cache_len,
+                max_batch_size=input_ids.shape[0],
+                seqlen_offset=0,
+                batch_size_offset=0,
+                kv_block_map={},
+                lengths_per_sample=None,
+            )
+        else:
+            # assume that `kv_cache` has cached all tokens up to the last token in `input_ids`
+            kv_cache.seqlen_offset = input_ids.shape[1] - 1
+            input_ids = cast(torch.LongTensor, input_ids[:, -1].unsqueeze(-1))
+
+        return {  # to be passed to forward()
+            "input_ids": input_ids,
+            "kv_cache": kv_cache,
+            "key_padding_mask": key_padding_mask,
+        }
+
+
+class Embedding(nn.Module):
+    """Token embedding with dropout from Phi2."""
+
+    def __init__(
+        self,
+        vocab_size: int,
+        d_embedding: int,
+        embd_pdrop: float,
+    ) -> None:
+        super().__init__()
+        self.embeddings = nn.Embedding(vocab_size, d_embedding)
+        self.dropout = nn.Dropout(embd_pdrop)
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor,  # dim: (batch_size, seq_len)
+    ) -> torch.FloatTensor:
+        x = self.embeddings(  # dim: (batch_size, seq_len, d_embedding)
+            input_ids.view(-1, input_ids.size()[-1])
+        )
+        x = self.dropout(x)
+        return x
+
+
+class Phi2Model(Phi2PreTrainedModel):
+    def __init__(self, config: Phi2Config) -> None:
+        super().__init__(config)
+        self.embedding = Embedding(
+            vocab_size=config.vocab_size,
+            d_embedding=config.d_embedding,
+            embd_pdrop=config.embd_pdrop,
+        )
+        self.parallel_blocks = nn.ModuleList([
+            ParallelAttentionBlock(
+                resid_pdrop=config.resid_pdrop,
+                layer_norm_epsilon=config.layer_norm_epsilon,
+                d_embedding=config.d_embedding,
+                n_attn_heads=config.n_attn_heads,
+                block_n=i,
+                initial_cos_sin_cache_len=config.initial_cos_sin_cache_len,
+                attn_pdrop=config.attn_pdrop,
+                use_flash_rotary=config.use_flash_rotary,
+                use_flash_attn=config.use_flash_attn,
+                use_fused_dense=config.use_fused_dense,
+                checkpointing=config.checkpointing,
+            )
+            for i in range(config.n_blocks)
+        ])
+        self.gradient_checkpointing_disable()  # https://github.com/cybertronai/gradient-checkpointing - I think this is turned off due to flash attention?
+        self.post_init()  # calls self._init_weights() for all modules
+
+    """
+    def get_input_embeddings(self) -> nn.Embedding:
+        return self.embedding.embeddings
+
+    def set_input_embeddings(self, new_embeddings: nn.Embedding) -> None:
+        self.embedding.embeddings = new_embeddings
+    """
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor,
+        kv_cache: KVCache | None = None,
+        key_padding_mask: torch.BoolTensor | None = None,
+    ) -> torch.FloatTensor:
+        x = self.embedding(input_ids)
+        for block in self.parallel_blocks:
+            x = block(
+                x,
+                kv_cache=kv_cache,
+                key_padding_mask=key_padding_mask,
+            )
+        return x
+
+
+class Phi2ModelForCausalLM(Phi2PreTrainedModel):
+    def __init__(self, config: Phi2Config) -> None:
+        super().__init__(config)
+        self.pretrained_model = Phi2Model(config)
+        self.layer_norm = nn.LayerNorm(config.d_embedding, eps=config.layer_norm_epsilon)
+        self.linear = nn.Linear(config.d_embedding, config.vocab_size)
+        self.loss_fn = nn.CrossEntropyLoss()
+        self.post_init()  # calls self._init_weights() for all modules
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor,
+        kv_cache: KVCache | None = None,
+        key_padding_mask: torch.BoolTensor | None = None,
+        labels: torch.LongTensor | None = None,
+    ) -> CausalLMOutputWithPast:
+        x = self.pretrained_model(input_ids, kv_cache=kv_cache, key_padding_mask=key_padding_mask)
+        x = self.layer_norm(x)
+        logits = self.linear(x).to(torch.float32)
+        loss = (
+            self.loss_fn(logits.view(-1, logits.size(-1)), labels.view(-1))
+            if labels is not None
+            else None
+        )
+        return CausalLMOutputWithPast(loss=loss, logits=logits)