Initial Commit

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "python.formatting.provider": "black"
+}

app.py

@@ -0,0 +1,143 @@
+# Copyright 2022 Tristan Behrens.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+
+from flask import Flask, render_template, request, send_file, jsonify, redirect, url_for
+from PIL import Image
+import os
+import io
+import random 
+import base64
+import torch
+import wave
+from source.logging import create_logger
+from source.tokensequence import token_sequence_to_audio, token_sequence_to_image
+from source import constants
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+logger = create_logger(__name__)
+
+# Load the auth-token from authtoken.txt.
+# auth_token = None
+# with open("authtoken.txt", "r") as authtoken_file:
+#     auth_token = authtoken_file.read().strip()
+auth_token = os.getenv("authtoken")
+
+# Loading the model and its tokenizer.
+logger.info("Loading tokenizer and model...")
+tokenizer = AutoTokenizer.from_pretrained("ai-guru/lakhclean_mmmtrack_4bars_d-2048", use_auth_token=auth_token)
+model = AutoModelForCausalLM.from_pretrained("ai-guru/lakhclean_mmmtrack_4bars_d-2048", use_auth_token=auth_token)
+logger.info("Done.")
+
+# Create the app.
+logger.info("Creating app...")
+app = Flask(__name__)
+logger.info("Done.")
+
+# Route for the loading page.
+@app.route("/")
+def index():
+    return render_template(
+        "composer.html",
+        compose_styles = constants.get_compose_styles_for_ui(),
+        densities=constants.get_densities_for_ui(),
+        temperatures=constants.get_temperatures_for_ui(),
+    )
+
+@app.route("/compose", methods=["POST"])
+def compose():
+
+    # Get the parameters as JSON.
+    params = request.get_json()
+    music_style = params["music_style"]
+    density = params["density"]
+    temperature = params["temperature"]
+
+    instruments = constants.get_instruments(music_style)
+    density = constants.get_density(density)
+    temperature = constants.get_temperature(temperature)
+    print(f"instruments: {instruments} density: {density} temperature: {temperature}")
+
+    # Generate with the given parameters.
+    logger.info(f"Generating token sequence...")
+    generated_sequence = generate_sequence(instruments, density, temperature)
+    logger.info(f"Generated token sequence: {generated_sequence}")
+
+    # Get the audio data as a array of int16.
+    logger.info("Generating audio...")
+    sample_rate, audio_data = token_sequence_to_audio(generated_sequence)
+    logger.info(f"Done. Audio data: {len(audio_data)}")
+
+    # Encode the audio-data as wave file in memory. Use the wave module.
+    audio_data_bytes = io.BytesIO()
+    wave_file = wave.open(audio_data_bytes, "wb")
+    wave_file.setframerate(sample_rate)
+    wave_file.setnchannels(1)
+    wave_file.setsampwidth(2)
+    wave_file.writeframes(audio_data)
+    wave_file.close()
+
+    # Return the audio-data as a base64-encoded string.
+    audio_data_bytes.seek(0)
+    audio_data_base64 = base64.b64encode(audio_data_bytes.read()).decode("utf-8")
+    audio_data_bytes.close()
+
+    # Convert the audio data to an PIL image.
+    image = token_sequence_to_image(generated_sequence)
+
+    # Save PIL image to harddrive as PNG.
+    logger.debug(f"Saving image to harddrive... {type(image)}")
+    image_file_name = "compose.png"
+    image.save(image_file_name, "PNG")
+    
+    # Save image to virtual file.
+    img_io = io.BytesIO()
+    image.save(img_io, 'PNG', quality=70)
+    img_io.seek(0)
+
+    # Return the image as a base64-encoded string.
+    image_data_base64 = base64.b64encode(img_io.read()).decode("utf-8")
+    img_io.close()
+
+    # Return.
+    return jsonify({
+        "tokens": generated_sequence,
+        "audio": "data:audio/wav;base64," + audio_data_base64, 
+        "image": "data:image/png;base64," + image_data_base64,
+        "status": "OK"
+    })
+
+def generate_sequence(instruments, density, temperature):
+
+    instruments = instruments[::]
+    random.shuffle(instruments)
+
+    generated_ids = tokenizer.encode("PIECE_START", return_tensors="pt")[0]
+    
+    for instrument in instruments:
+        more_ids = tokenizer.encode(f"TRACK_START INST={instrument} DENSITY={density}", return_tensors="pt")[0]
+        generated_ids = torch.cat((generated_ids, more_ids))
+        generated_ids = generated_ids.unsqueeze(0)
+
+        generated_ids = model.generate(
+            generated_ids,
+            max_length=2048,
+            do_sample=True,
+            temperature=temperature,
+            eos_token_id=tokenizer.encode("TRACK_END")[0] 
+        )[0]
+
+    generated_sequence = tokenizer.decode(generated_ids)
+    return generated_sequence

packages.txt

@@ -0,0 +1,6 @@
+libfluidsynth1
+fluid-soundfont-gm
+build-essential
+libasound2-dev
+libjack-dev
+ffmpeg

requirements.txt

@@ -0,0 +1,7 @@
+transformers
+tokenizers
+datasets
+flask
+torch
+pyfluidsynth
+note_seq

source/constants.py

@@ -0,0 +1,85 @@
+compose_styles_config = {
+    "piano": {
+        "readable": "Piano",
+        "instruments": ["1"],
+    },
+    "chamber": {
+        "readable": "Chamber Music",
+        "instruments": ["0", "40", "42"],
+    },
+    "rock_and_metal": {
+        "readable": "Rock and Metal",
+        "instruments": ["DRUMS", "30", "34"],
+    },
+    "synth": {
+        "readable": "Synthesizer",
+        "instruments": ["DRUMS", "38", "80"],
+    },
+    "church": {
+        "readable": "Church",
+        "instruments": ["19", "52"],
+    },
+    "timpani_strings_harp": {
+        "readable": "Timpani, Contrabass, Harp",
+        "instruments": ["47", "43", "46"],
+    },
+    "country": {
+        "readable": "Country",
+        "instruments": ["DRUMS", "22", "32", "25"],
+    },
+}
+
+densities_config = {
+    "low": {
+        "readable": "Low",
+        "density": 4,
+    },
+    "medium": {
+        "readable": "Medium",
+        "density": 6,
+    },
+    "high": {
+        "readable": "High",
+        "density": 8,
+    },
+}
+
+temperatures_config = {
+    "low": {
+        "readable": "Low",
+        "temperature": 0.5,
+    },
+    "medium": {
+        "readable": "Medium",
+        "temperature": 0.75,
+    },
+    "high": {
+        "readable": "High",
+        "temperature": 1.0,
+    },
+    "very_high": {
+        "readable": "Very High",
+        "temperature": 1.25,
+    },
+}
+
+def get_compose_styles_for_ui():
+    compose_styles = [[key, compose_styles_config[key]["readable"]] for key, value in compose_styles_config.items()]
+    return compose_styles
+
+def get_densities_for_ui():
+    densities = [[key, densities_config[key]["readable"]] for key, value in densities_config.items()]
+    return densities
+
+def get_temperatures_for_ui():
+    temperatures = [[key, temperatures_config[key]["readable"]] for key, value in temperatures_config.items()]
+    return temperatures
+
+def get_instruments(key):
+    return compose_styles_config[key]["instruments"]
+
+def get_density(key):
+    return densities_config[key]["density"]
+
+def get_temperature(key):
+    return temperatures_config[key]["temperature"]

source/logging.py

@@ -0,0 +1,45 @@
+# Copyright 2021 Tristan Behrens.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+
+import logging
+
+loggers_dict = {}
+
+def create_logger(name:str):
+    global loggers_dict
+    if name in loggers_dict:
+        return loggers_dict[name]
+    else:  
+        logger = logging.getLogger(name)
+        loggers_dict[name] = logger
+        logger.setLevel(logging.DEBUG)
+        handler = logging.StreamHandler()
+        handler.setLevel(logging.DEBUG)
+        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        logger.propagate = False
+        return logger
+
+def set_log_level(name, level):
+    logger_names = []
+    if name == "all":
+        logger_names = list(loggers_dict.keys())
+    else:
+        logger_names = [name]
+    for name in logger_names:
+        logger = loggers_dict[name]
+        logger.setLevel(level)

source/tokensequence.py

@@ -0,0 +1,160 @@
+import note_seq
+import numpy as np
+from PIL import Image
+
+NOTE_LENGTH_16TH_120BPM = 0.25 * 60 / 120
+BAR_LENGTH_120BPM = 4.0 * 60 / 120
+SAMPLE_RATE=44100
+
+def token_sequence_to_audio(token_sequence):
+    note_sequence = token_sequence_to_note_sequence(token_sequence)
+    synth = note_seq.midi_synth.fluidsynth
+    array_of_floats = synth(note_sequence, sample_rate=SAMPLE_RATE)
+    note_plot = note_seq.plot_sequence(note_sequence, False)
+    array_of_floats /=1.414
+    array_of_floats *= 32767
+    int16_data = array_of_floats.astype(np.int16)
+    return SAMPLE_RATE, int16_data
+
+def token_sequence_to_image(token_sequence):
+    
+    note_sequence = token_sequence_to_note_sequence(token_sequence)
+
+    # Find minumum and maximum pitch.
+    min_pitch = 128
+    max_pitch = 0
+    for note in note_sequence.notes:
+        if note.pitch < min_pitch:
+            min_pitch = note.pitch
+        if note.pitch > max_pitch:
+            max_pitch = note.pitch
+
+    image_height = max_pitch - min_pitch + 1
+    image_width = int(16 * 4)
+
+    color = (150, 15, 15)
+
+    # Indicate that this bar was used for composition.
+    image = Image.new("RGB", (image_width, image_height), color)
+
+    colors = [(0, 0, 0), (255, 255, 255), (255, 0, 0), (0, 255, 0), (0, 0, 255), (255, 255, 0), (255, 0, 255), (0, 255, 255)]
+
+    instrument_to_color_index = {}
+
+    # Draw the notes.
+    for note in note_sequence.notes:
+        x = int(note.start_time / note_sequence.total_time * image_width)
+        y = note.pitch - min_pitch
+        width = int((note.end_time - note.start_time) / note_sequence.total_time * image_width)
+        height = 1
+
+        if note.instrument not in instrument_to_color_index:
+            instrument_to_color_index[note.instrument] = len(instrument_to_color_index)
+        color_index = instrument_to_color_index[note.instrument]
+        color = colors[color_index]
+
+        #color = (255, 255, 255)
+        image.paste(color, (x, y, x + width, y + height))
+
+    # Rescale and rotate.
+    factor = 8
+    image = image.resize((image_width * factor, image_height * factor), Image.NEAREST)
+    image = image.transpose(Image.FLIP_TOP_BOTTOM)
+
+    return image
+    
+    
+    
+
+def token_sequence_to_note_sequence(token_sequence, use_program=True, use_drums=True, instrument_mapper=None, only_piano=False):
+    if isinstance(token_sequence, str):
+        token_sequence = token_sequence.split()
+    note_sequence = empty_note_sequence()
+
+    # Render all notes.
+    current_program = 1
+    current_is_drum = False
+    current_instrument = 0
+    track_count = 0
+    for token_index, token in enumerate(token_sequence):
+
+        if token == "PIECE_START":
+            pass
+        elif token == "PIECE_END":
+            print("The end.")
+            break
+        elif token == "TRACK_START":
+            current_bar_index = 0
+            track_count += 1
+            pass
+        elif token == "TRACK_END":
+            pass
+        elif token.startswith("INST"):
+            instrument = token.split("=")[-1]
+            if instrument != "DRUMS" and use_program:
+                if instrument_mapper is not None:
+                    if instrument in instrument_mapper:
+                        instrument = instrument_mapper[instrument]
+                current_program = int(instrument)
+                current_instrument = track_count
+                current_is_drum = False
+            if instrument == "DRUMS" and use_drums:
+                current_instrument = 0
+                current_program = 0
+                current_is_drum = True
+        elif token == "BAR_START":
+            current_time = current_bar_index * BAR_LENGTH_120BPM
+            current_notes = {}
+        elif token == "BAR_END":
+            current_bar_index += 1
+            pass
+        elif token.startswith("NOTE_ON"):
+            pitch = int(token.split("=")[-1])
+            note = note_sequence.notes.add()
+            note.start_time = current_time
+            note.end_time = current_time + 4 * NOTE_LENGTH_16TH_120BPM
+            note.pitch = pitch
+            note.instrument = current_instrument
+            note.program = current_program
+            note.velocity = 80
+            note.is_drum = current_is_drum
+            current_notes[pitch] = note
+        elif token.startswith("NOTE_OFF"):
+            pitch = int(token.split("=")[-1])
+            if pitch in current_notes:
+                note = current_notes[pitch]
+                note.end_time = current_time
+        elif token.startswith("TIME_DELTA"):
+            delta = float(token.split("=")[-1]) * NOTE_LENGTH_16TH_120BPM
+            current_time += delta
+        elif token.startswith("DENSITY="):
+            pass
+        elif token == "[PAD]":
+            pass
+        else:
+            pass
+
+    # Make the instruments right.
+    instruments_drums = []
+    for note in note_sequence.notes:
+        pair = [note.program, note.is_drum]
+        if pair not in instruments_drums:
+            instruments_drums += [pair]
+        note.instrument = instruments_drums.index(pair)
+
+    if only_piano:
+        for note in note_sequence.notes:
+            if not note.is_drum:
+                note.instrument = 0
+                note.program = 0
+
+    note_sequence.total_time = current_time
+
+    return note_sequence
+
+def empty_note_sequence(qpm=120.0, total_time=0.0):
+    note_sequence = note_seq.protobuf.music_pb2.NoteSequence()
+    note_sequence.tempos.add().qpm = qpm
+    note_sequence.ticks_per_quarter = note_seq.constants.STANDARD_PPQ
+    note_sequence.total_time = total_time
+    return note_sequence

startserver.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/sh
+
+export FLASK_ENV=development 
+export FLASK_DEBUG=1
+flask run

static/script.js

@@ -0,0 +1,67 @@
+function compose() {
+
+    // Get the value of the select with id selected_midi_program.
+    var selected_music_style = document.getElementById("selected_music_style").value;
+    var selected_density = document.getElementById("selected_density").value;
+    var selected_temperature = document.getElementById("selected_temperature").value;
+
+    var xhr = new XMLHttpRequest();
+    xhr.open("POST", "/compose", true);
+    xhr.setRequestHeader("Content-Type", "application/json");
+    xhr.send(JSON.stringify({
+        music_style: selected_music_style,
+        density: selected_density,
+        temperature: selected_temperature
+    }));
+
+    xhr.onreadystatechange = function() {
+        if (xhr.readyState == 4 && xhr.status == 200) {
+            var response = JSON.parse(xhr.responseText);
+            console.log(response);
+
+            if (response.status == "OK") {
+
+                // Replace the inner html of the div with id token_sequence with the token sequence.
+                document.getElementById("token_sequence").innerHTML = response.tokens;
+
+                // Replace the source of the audio element with id audio..
+                var audio = document.getElementById("audio_id");
+                audio.src = response.audio;
+
+                // Replace the source of the image element with id image.
+                var image = document.getElementById("image_id");
+                image.src = response.image;
+                
+            }
+            else {
+                alert("Error: " + response.error);
+            }
+        }
+    }
+}
+
+
+function post_command(command_name, command_parameters, reload) {
+    var xhr = new XMLHttpRequest();
+    xhr.open("POST", "/command", true);
+    xhr.setRequestHeader("Content-Type", "application/json");
+    xhr.send(JSON.stringify({command_name: command_name, command_parameters: command_parameters}));
+
+    xhr.onreadystatechange = function() {
+        if (xhr.readyState == 4 && xhr.status == 200) {
+            var response = JSON.parse(xhr.responseText);
+            if (response.status == "OK") {
+
+                // Reload the page if requested.
+                if (reload) {
+                    console.log("Reloading.");
+                    load_cells();
+                }
+                
+            }
+            else {
+                alert("Error: " + response.error);
+            }
+        }
+    }
+}

templates/composer.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+    <title>Composer</title>
+    <script src="https://kit.fontawesome.com/d6e88637ad.js" crossorigin="anonymous"></script>
+    <link rel="stylesheet" href="/static/style.css" />
+    <script src="{{ url_for('static', filename='script.js') }}"></script>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
+  </head>
+
+    <body>
+        <div class="container">
+
+            <div>Style</div>
+            <select id="selected_music_style">
+                {%for compose_style in compose_styles%}
+                    <option value="{{compose_style[0]}}">{{compose_style[1]}}</option>
+                {%endfor%}
+            </select>
+
+            <div>Density</div>
+            <select id="selected_density">
+                {%for density in densities%}
+                    <option value="{{density[0]}}">{{density[1]}}</option>
+                {%endfor%}
+            </select>
+
+            <div>Temperature</div>
+            <select id="selected_temperature">
+                {%for temperature in temperatures%}
+                    <option value="{{temperature[0]}}">{{temperature[1]}}</option>
+                {%endfor%}
+            </select>
+
+            <audio id="audio_id" controls="controls" autobuffer="autobuffer" autoplay="autoplay">
+                <source src="" />
+            </audio>
+
+            <img id="image_id" src="" />
+        
+            <div class="item btn" command="compose" style="cursor: pointer;" onclick="compose()">Compose</div>
+
+            <div id="token_sequence">PIECE_START</div>
+
+        </div>
+    </body>
+</html>