Upload 14 files

hparams.py

@@ -0,0 +1,29 @@
+import librosa
+
+### Audio Hyperparameters ###
+sr = 44100
+
+lf_params = {
+  'min_f': librosa.note_to_hz('c0'),
+  'max_f': 4100,
+  'bins_per_octave': 24,
+  'gamma': 20
+}
+
+
+hf_params = {
+  'min_f': 4100,
+  'max_f': 16350,
+  'bins_per_octave': 96,
+  'gamma': 0
+}
+
+### Network Hyperparameters ###
+
+n_channels = 1
+chunk_size = 512
+frequency_bins = 192
+batch_size = 32
+learning_rate = 0.0001
+epochs = 35
+inference_batch_size = 4

multiresunet_model.py

@@ -0,0 +1,161 @@
+import tensorflow as tf
+
+def Downsampling(x, filters, kernel_size = (5,5), padding = 'same', stride = 2, multires = False):
+    '''
+    Downsampling Block
+    
+    Arguments:
+        x : input layer (tf.keras.layer)
+        filters : number of filters (int)
+        kernel_size : kernel dimensions (tuple or int), default (5,5)
+        padding : padding type for convolution (string), default same
+        stride : stride for convolution (tuple or int), default 2
+    
+    Returns:
+        output : output layer (tf.keras.layer)
+    '''
+    if multires == False:
+      conv = tf.keras.layers.Conv2D(kernel_size = kernel_size, filters = filters, strides = stride, padding = padding,data_format = "channels_last")(x)
+    elif multires == True:
+      conv = tf.keras.layers.Conv2D(kernel_size = kernel_size, filters = filters//2, strides = stride, padding = padding,data_format = "channels_last")(x)
+      conv3 = tf.keras.layers.Conv2D(kernel_size = (3,3), filters = filters//4, strides = stride, padding = padding,data_format = "channels_last")(x)
+      conv7 = tf.keras.layers.Conv2D(kernel_size = (7,7), filters = filters//4, strides = stride, padding = padding,data_format = "channels_last")(x)
+      conv = tf.keras.layers.Concatenate()([conv, conv3, conv7])
+    bn = tf.keras.layers.BatchNormalization()(conv)
+    output = tf.keras.layers.LeakyReLU(0.2)(bn)
+
+    return output
+
+def Upsampling(x , y, filters, res_filts, kernel_size = (5,5), padding = 'same', stride = 2, dropout = 'False', resblock = True, se_block = False):
+    '''
+    Upsampling Block
+    
+    Arguments:
+        x : input layer (tf.keras.layer)
+        y : residual connection layer (tf.keras.layer)
+        filters : number of filters (int)
+        kernel_size : kernel dimensions (tuple or int), default (5,5)
+        padding : padding type for convolution (string), default same
+        stride : stride for convolution (tuple or int), default 2
+        dropout : dropout (boolean), default False
+    
+    Returns:
+        output : output layer (tf.keras.layer)
+    '''
+    
+    conv = tf.keras.layers.Conv2DTranspose(kernel_size = kernel_size, filters = filters, strides = stride, padding = padding, data_format = "channels_last")(x)
+    act = tf.keras.layers.ReLU()(conv)
+    output = tf.keras.layers.BatchNormalization()(act)
+    if dropout == 'True':
+      output = tf.keras.layers.Dropout(0.5)(output)
+    if y is not None:
+      if resblock is True:
+        y = ResBlock(y, depth = 2, filters = res_filts)
+      output = tf.keras.layers.Concatenate()([y, output])
+    if se_block is True:
+      output = SE_Block(output, r = 16)
+    return output
+
+def ResBlock(x, filters, depth = 2, kernel_size = (5,5), padding = 'same', method = 'concat', se_block = False):
+      '''
+    ResNet Block
+    
+    Arguments:
+        x : input layer (tf.keras.layer)
+        depth : number of layers in ResBlock
+        filters : number of filters (int)
+        kernel_size : kernel dimensions (tuple or int), default (5,5)
+        padding : padding type for convolution (string), default same
+        dropout : dropout (boolean), default False
+    
+    Returns:
+        output : output layer (tf.keras.layer)
+    '''
+
+      conv = tf.keras.layers.Conv2D(kernel_size = kernel_size, filters = filters, padding = padding, data_format = "channels_last")(x)
+      conv = tf.keras.layers.ReLU()(conv)
+      conv = tf.keras.layers.BatchNormalization()(conv)
+      for i in range(0,depth-1):
+        conv = tf.keras.layers.Conv2D(kernel_size = kernel_size, filters = filters, padding = padding, data_format = "channels_last")(conv)
+        conv = tf.keras.layers.ReLU()(conv)
+        conv = tf.keras.layers.BatchNormalization()(conv)
+      if method == 'add':
+        output = tf.keras.layers.Add()([x, conv])
+      elif method == 'concat':
+        output = tf.keras.layers.Concatenate()([x, conv])
+      
+      output = tf.keras.layers.ReLU()(output)
+
+      if se_block is True:
+       output = SE_Block(output, r = 16)
+
+      return output
+
+def SE_Block(x, r = 16):
+
+  '''
+    Squeeze and Excitation Block
+    Assumes channel_last format
+    
+    Arguments:
+        x : input layer (tf.keras.layer)
+        r : reduction ratio for first FC layer
+    
+    Returns:
+        output : output layer (tf.keras.layer)
+  '''
+  filters = x.shape[-1]
+  pool = tf.keras.layers.GlobalAveragePooling2D(data_format='channels_last')(x)
+  fc1 = tf.keras.layers.Dense(int(filters/r))(pool)
+  fc1 = tf.keras.layers.ReLU()(fc1)
+  fc2 = tf.keras.layers.Dense(filters)(fc1)
+  fc2 = tf.keras.layers.Activation('sigmoid')(fc2)
+  output = tf.keras.layers.Reshape([1,1,filters])(fc2)
+
+  output = tf.keras.layers.Multiply()([x,output])
+
+  return output
+
+def Steminator(input_shape = (256,128,1), kernel_size = (5,5), feature_maps = 8, multires = True, resblock = True, se_block = True):
+
+    '''
+    MultiResUnet Network Builder - Steminator
+    
+    Arguments:
+        input_shape : input shape (tuple)
+        depth : number of layers in ResBlock
+        feature_maps : number of initial filters (int)
+        kernel_size : kernel dimensions (tuple or int), default (5,5)
+        multires : use multi-res Unet (boolean), default True
+        resblock : use resblock residual connections (boolean), default True
+    
+    Returns:
+        model : tf.keras Neural net model (tf.keras.Model)
+    '''
+
+    cqt_input = tf.keras.Input(shape=input_shape)
+    
+    ds_0 = Downsampling(cqt_input, filters = feature_maps*2, multires = multires)
+    ds_1 = Downsampling(ds_0, filters = feature_maps*4, multires = multires)
+    ds_2 = Downsampling(ds_1, filters = feature_maps*8, multires = multires)
+    ds_3 = Downsampling(ds_2, filters = feature_maps*16, multires = multires)
+    ds_4 = Downsampling(ds_3, filters = feature_maps*32, multires = multires)
+    ds_5 = Downsampling(ds_4, filters = feature_maps*64, multires = multires)
+    
+    us_0 = Upsampling(ds_5,ds_4,filters = feature_maps*32, res_filts = feature_maps, dropout = 'True', resblock = resblock)
+    us_1 = Upsampling(us_0,ds_3,filters = feature_maps*16, res_filts = feature_maps*2, dropout = 'True', resblock = resblock)
+    us_2 = Upsampling(us_1,ds_2,filters = feature_maps*8, res_filts = feature_maps*4, dropout = 'True', resblock = resblock)
+    us_3 = Upsampling(us_2,ds_1,filters = feature_maps*4, res_filts = feature_maps*8, resblock = resblock)
+    us_4 = Upsampling(us_3,ds_0,filters = feature_maps*2, res_filts = feature_maps*16, resblock = resblock, se_block = False)
+    us_5 = Upsampling(us_4,None,filters = feature_maps, res_filts = feature_maps*32, resblock = resblock, se_block = False)
+
+    
+    mask = tf.keras.layers.Conv2D(kernel_size = (1,1), filters = 1,activation='relu', padding = 'same',data_format="channels_last")(us_5) #original network kernel_size = (1,1)
+
+    outputs = tf.keras.layers.Multiply()([cqt_input,mask])
+  
+    model = tf.keras.Model(inputs = cqt_input, outputs = outputs, name='Steminator')
+
+    #model.summary()
+
+    return model

preprocess_data.py

@@ -0,0 +1,115 @@
+import argparse
+import glob
+
+import numpy as np
+import librosa
+from essentia.standard import (NSGConstantQ, 
+    NSGIConstantQ)
+
+import hparams
+import utils
+
+def parse_files(path, source):
+
+  if source == 'mixture':
+    path = path + 'Mixtures/Dev/*/' + str(source) + '.wav'
+    paths = sorted(glob.glob(path))
+  else:
+    path = path + 'Sources/Dev/*/' + str(source) + '.wav'
+    paths = sorted(glob.glob(path))
+  return paths
+
+def forward_transform(y, min_f, max_f, bpo, gamma):
+  # Parameters
+  params = {
+            # Backward transform needs to know the signal size.
+            'inputSize': y.size,
+            'minFrequency': min_f,
+            'maxFrequency': max_f,
+            'binsPerOctave': bpo,
+            # Minimum number of FFT bins per CQ channel.
+            'minimumWindow': 4,
+            'gamma': gamma
+          }
+
+
+  # Forward and backward transforms
+  constantq, dcchannel, nfchannel = NSGConstantQ(**params)(y)
+
+  return constantq, dcchannel, nfchannel
+
+def backward_transform(c, dc, nf, orig_size, min_f, max_f, bpo, gamma):
+  # Parameters
+  params = {
+            # Backward transform needs to know the signal size.
+            'inputSize': orig_size,
+            'minFrequency': min_f,
+            'maxFrequency': max_f,
+            'binsPerOctave': bpo,
+            # Minimum number of FFT bins per CQ channel.
+            'minimumWindow': 4,
+            'gamma': gamma
+          }
+
+
+  # Forward and backward transforms
+  y = NSGIConstantQ(**params)(c, dc, nf)
+
+  return y
+
+
+def make_chunks(c):
+  cqt = np.abs(c).astype(np.float16)
+  cqt = np.asfortranarray(cqt)
+  padded_cqt = librosa.util.fix_length(cqt,hparams.chunk_size*np.ceil(cqt.shape[-1]/hparams.chunk_size).astype(int))
+  framed_cqt = librosa.util.frame(padded_cqt,hparams.chunk_size,hparams.chunk_size)
+  samples = np.transpose(framed_cqt,(2,0,1))
+  cqt_input = np.expand_dims(samples,-1)
+  return cqt_input
+
+if __name__ == '__main__':
+  args = argparse.ArgumentParser()
+
+  args.add_argument('Path',metavar='path',type=str,help='Path to DSD100')
+  args.add_argument('Source',metavar='source',type=str,help='Desired source to preprocess for separation. Use mixture to preprocess the mixtures')
+  args.add_argument('Output_path',metavar='output_path',type=str,help='Output path for the pikled spectrograms')
+
+  args = args.parse_args()
+  path = args.Path
+  source = args.Source
+  outpath = args.Output_path
+
+  if path[-1] != '/':
+    path = path + '/'
+  if outpath[-1] != '/':
+    outpath = outpath + '/'
+
+
+  files = parse_files(path, source)
+  mag_lf_array = []
+  mag_hf_array = []
+
+  for i in range(0,len(files)):
+    print(files[i])
+    y, sr = librosa.load(files[i], hparams.sr, mono = True)
+    C_lf,_,_ = forward_transform(y,hparams.lf_params['min_f'],hparams.lf_params['max_f'],hparams.lf_params['bins_per_octave'], hparams.lf_params['gamma'])
+    C_hf,_,_ = forward_transform(y,hparams.hf_params['min_f'],hparams.hf_params['max_f'],hparams.hf_params['bins_per_octave'], hparams.hf_params['gamma'])
+    c_lf = make_chunks(C_lf)
+    c_hf = make_chunks(C_hf)
+    mag_lf_array.append(c_lf)
+    mag_hf_array.append(c_hf)
+    if  i == 1:
+      break
+
+
+  mag_lf = utils.list_to_array(mag_lf_array)
+  mag_hf = utils.list_to_array(mag_hf_array)
+
+
+  filename_lf = source + '_lf.npy'
+  filename_hf = source + '_hf.npy'
+  utils.pickle(mag_lf, outpath, filename_lf)
+  utils.pickle(mag_hf, outpath, filename_hf)
+
+
+

pretrained_models/bass_hf.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ee25f613e06405327650d09c32a218e8d72ff1a657492acc12729d4d73e27f7
+size 133864672

pretrained_models/bass_lf.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b81709b811e7d605b2f1c8107075e3bd6e3cb3bfce4e51922d26edc06fda8844
+size 133865952

pretrained_models/drums_hf.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da3a8d08e4b4f6783fcc8601fad48e8fea3ee3077cc8ec91b34b34d2946b547b
+size 133864672

pretrained_models/drums_lf.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27378ddab7e14033eff4f4de3baf5be3d7da12c0fd02ce90cf127a02df4d6b17
+size 133864672

pretrained_models/other_hf.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41a29798ce7487d180b8afb59e1197336efb8467f86d76f531de97e81c913bef
+size 133864672

pretrained_models/other_lf.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ae2ac9c3c26c2f9544e7f991ca01a2daa6c61e4350d9594a3eb16785092a699
+size 133864672

pretrained_models/vocals_hf.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76232de963eba108bf4a1a41f97c7f9a361b29dc5609f902c0d4b90beb0c32e0
+size 133864672

pretrained_models/vocals_lf.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd29e527a5508b5f0e6d618261ab79f16d3c839d8ff642068d40095ab0364dd4
+size 133862488

separate.py

@@ -0,0 +1,79 @@
+import argparse
+
+import hparams
+import utils
+import multiresunet_model
+import preprocess_data
+
+import tensorflow as tf
+import numpy as np
+import librosa
+import torchaudio
+import torch
+
+if __name__ == '__main__':
+  args = argparse.ArgumentParser()
+
+  args.add_argument('Path',metavar='path',type=str,help='Path to audio track to be separated')
+  args.add_argument('Source',metavar='source',type=str,help='Desired source to separate')
+  args.add_argument('Model_path', metavar='path_to_model',type=str,help='Path to saved models')
+  args.add_argument('Output_path', metavar='output_path',type=str,help='Output path for separated audio')
+
+
+  ### Parse args ###
+  args = args.parse_args()
+  path_to_audio = args.Path
+  source = args.Source
+  path_to_model = args.Model_path
+  output_path = args.Output_path + source + '.wav' 
+
+  ### Load models ###
+  model_lf = tf.keras.models.load_model(path_to_model + source + '_lf.h5')
+  model_hf = tf.keras.models.load_model(path_to_model + source + '_hf.h5')
+
+  ### Load audio track ###
+  y, sr = librosa.load(path_to_audio, hparams.sr, mono = True)
+
+  ### Perform CQT transform on the audio ###
+  C_lf,dc_lf,nf_lf = preprocess_data.forward_transform(y,hparams.lf_params['min_f'],hparams.lf_params['max_f'],hparams.lf_params['bins_per_octave'], hparams.lf_params['gamma'])
+  C_hf,dc_hf,nf_hf = preprocess_data.forward_transform(y,hparams.hf_params['min_f'],hparams.hf_params['max_f'],hparams.hf_params['bins_per_octave'], hparams.hf_params['gamma'])
+  
+  dc_lf[:] = 0
+  dc_hf[:] = 0
+  nf_lf[:] = 0
+  nf_hf[:] = 0
+
+  phase_lf = np.angle(C_lf)
+  phase_hf = np.angle(C_hf)
+
+  ### Batch Input ###
+  c_lf = preprocess_data.make_chunks(C_lf)
+  c_hf = preprocess_data.make_chunks(C_hf)
+
+  ### Separate LF and HF ###
+  c_lf = model_lf.predict(c_lf,batch_size = hparams.inference_batch_size)
+  c_hf = model_hf.predict(c_hf,batch_size = hparams.inference_batch_size)
+
+  ### Reshape Model Output ###
+  mag_lf = np.hstack(c_lf[:,:,:,0])[:,:phase_lf.shape[-1]]
+  mag_hf = np.hstack(c_hf[:,:,:,0])[:,:phase_hf.shape[-1]]
+  c_lf = mag_lf * np.math.e**(phase_lf*1j)
+  c_hf = mag_hf * np.math.e**(phase_hf*1j)
+
+  ### Inverse CQT transform using the mixture phase information ###
+  y_lf_hat = preprocess_data.backward_transform(c_lf,dc_lf,nf_lf,y.shape[0],hparams.lf_params['min_f'],hparams.lf_params['max_f'],hparams.lf_params['bins_per_octave'], hparams.lf_params['gamma'])
+  y_hf_hat = preprocess_data.backward_transform(c_hf,dc_hf,nf_hf,y.shape[0],hparams.hf_params['min_f'],hparams.hf_params['max_f'],hparams.hf_params['bins_per_octave'], hparams.hf_params['gamma'])
+  y_hat = y_lf_hat + y_hf_hat
+
+  print(mag_lf.shape)
+  print(mag_hf.shape)
+
+  print(y_lf_hat.shape)
+  print(y_hf_hat.shape)
+
+  torchaudio.save(output_path, torch.from_numpy(np.expand_dims(y_hat,0)), hparams.sr)
+
+
+
+
+

train.py

@@ -0,0 +1,42 @@
+import argparse
+
+import hparams
+import utils
+import multiresunet_model
+
+import tensorflow as tf
+import numpy as np
+
+if __name__ == '__main__':
+  args = argparse.ArgumentParser()
+
+  args.add_argument('Path',metavar='path',type=str,help='Path to DSD100 pickled spectrograms. See preprocess_data.py for more details')
+  args.add_argument('Source',metavar='source',type=str,help='Desired source to separate')
+  args.add_argument('Spectrum',metavar='spectrum',type=str,help='Low (lf) or High (hf) frequencies training')
+  args.add_argument('Outpath',metavar='model_out_path',type=str,help='Path to save the model to')
+  
+  ### Parse Args ###
+  args = args.parse_args()
+  path = args.Path
+  source = args.Source
+  spectrum = args.Spectrum
+  output_path = args.Outpath
+  
+  ### Load Data ###
+  x = np.load(path + 'mixture_' + spectrum + '.npy')
+  y = np.load(path + source + '_' + spectrum + '.npy')
+
+  ### Construct model ###
+  model = multiresunet_model.Steminator((hparams.frequency_bins,hparams.chunk_size,hparams.n_channels))
+  optimizer = tf.keras.optimizers.Adam(lr = hparams.learning_rate)
+  model.compile(optimizer, loss='mean_absolute_error')
+
+  ### Training ###
+  model.fit(x,y,epochs = hparams.epochs, batch_size = hparams.batch_size)
+
+  ### Save model ###
+  model.save(output_path + source + '_' + spectrum + '.h5')
+
+
+
+

utils.py

@@ -0,0 +1,21 @@
+import numpy as np
+import os
+
+def create_dir(path):
+    if not os.path.exists(path):
+        try:
+            os.makedirs(path)
+        except OSError as e:
+            print('Could not create directory:' + path)
+
+def list_to_array(m):
+  M = m[0]
+  for i in range(1,len(m)):
+    M = np.concatenate((M,m[i]), axis = 0)
+
+  return M
+
+def pickle(array, path, filename):
+  create_dir(path)
+  np.save(path+filename, array)
+  return 0