Atotti/Qwen3-Omni-AudioTransformer

Qwen/Qwen3-Omni-30B-A3B-InstructのAudioEncoderを抽出した重みです。ライセンスは元モデルに従います。

詳細はQwen3-Omniテクニカルレポートを参照。

pip install git+https://github.com/huggingface/transformers

Usage

import soundfile as sf
import torch
from transformers import WhisperFeatureExtractor
from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import Qwen3OmniMoeAudioEncoder

model_id = "Atotti/Qwen3-Omni-AudioTransformer"
feature_extractor = WhisperFeatureExtractor.from_pretrained(model_id)
qwen_encoder = Qwen3OmniMoeAudioEncoder.from_pretrained(model_id, dtype=torch.bfloat16).eval()

wav, sr = sf.read("audio_16k.wav", dtype="float32")

features = feature_extractor(wav, sampling_rate=sr, return_tensors="pt", padding=False)
x = features["input_features"][0].to(qwen_encoder.dtype)
feat_len = torch.tensor([x.shape[-1]], dtype=torch.long)

with torch.no_grad():
    out = qwen_encoder(input_features=x, feature_lens=feat_len)
    hid = out.last_hidden_state # [T', D]
    print(hid.shape) # torch.Size([37, 2048])
    print(hid[0, :10]) # tensor([ 0.0088, -0.0032,  0.0154, -0.0138,  0.0040, -0.0037, -0.0017,  0.0180, -0.0149,  0.0051])

Qwen3OmniMoeAudioEncoderConfig {
  "activation_dropout": 0,
  "activation_function": "gelu",
  "architectures": [
    "Qwen3OmniMoeAudioEncoder"
  ],
  "attention_dropout": 0,
  "conv_chunksize": 500,
  "d_model": 1280,
  "downsample_hidden_size": 480,
  "dropout": 0,
  "dtype": "bfloat16",
  "encoder_attention_heads": 20,
  "encoder_ffn_dim": 5120,
  "encoder_layers": 32,
  "initializer_range": 0.02,
  "max_source_positions": 1500,
  "model_type": "qwen3_omni_moe_audio_encoder",
  "n_window": 50,
  "n_window_infer": 800,
  "num_hidden_layers": 32,
  "num_mel_bins": 128,
  "output_dim": 2048,
  "scale_embedding": false,
  "tf_legacy_loss": false,
  "transformers_version": "4.57.0.dev0",
  "use_bfloat16": false
}

Qwen3OmniMoeAudioEncoder(
  (positional_embedding): SinusoidsPositionEmbedding()
  (layers): ModuleList(
    (0-31): 32 x Qwen3OmniMoeAudioEncoderLayer(
      (self_attn): Qwen3OmniMoeAudioAttention(
        (k_proj): Linear(in_features=1280, out_features=1280, bias=True)
        (v_proj): Linear(in_features=1280, out_features=1280, bias=True)
        (q_proj): Linear(in_features=1280, out_features=1280, bias=True)
        (out_proj): Linear(in_features=1280, out_features=1280, bias=True)
      )
      (self_attn_layer_norm): LayerNorm((1280,), eps=1e-05, elementwise_affine=True)
      (activation_fn): GELUActivation()
      (fc1): Linear(in_features=1280, out_features=5120, bias=True)
      (fc2): Linear(in_features=5120, out_features=1280, bias=True)
      (final_layer_norm): LayerNorm((1280,), eps=1e-05, elementwise_affine=True)
    )
  )
  (ln_post): LayerNorm((1280,), eps=1e-05, elementwise_affine=True)
  (conv2d1): Conv2d(1, 480, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
  (conv2d2): Conv2d(480, 480, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
  (conv2d3): Conv2d(480, 480, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
  (conv_out): Linear(in_features=7680, out_features=1280, bias=False)
  (proj1): Linear(in_features=1280, out_features=1280, bias=True)
  (act): GELUActivation()
  (proj2): Linear(in_features=1280, out_features=2048, bias=True)
)