classi.py

# -*- coding: utf-8 -*-
"""Classi.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1tPr7FpJL4jUsqx4bEIBxBCmfVyngjYjz

# Classi (ver. 0.5)

***

Powered by tegridy-tools: https://github.com/asigalov61/tegridy-tools

***

WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect. https://www.nscai.gov/

***

#### Project Los Angeles

#### Tegridy Code 2022

***

# (GPU CHECK)
"""

#@title NVIDIA GPU check
!nvidia-smi

"""# (SETUP ENVIRONMENT)"""

#@title Install dependencies
!git clone https://github.com/asigalov61/Classi
!pip install torch
!pip install einops
!pip install torch-summary
!pip install sklearn
!pip install tqdm
!pip install matplotlib
!apt install fluidsynth #Pip does not work for some reason. Only apt works
!pip install midi2audio

# Commented out IPython magic to ensure Python compatibility.
#@title Import modules

print('=' * 70)
print('Loading core Classi modules...')

import os
import pickle
import random
import secrets
import statistics
from time import time as tim
import tqdm

print('=' * 70)
print('Loading main Classi modules...')
import torch

# %cd /content/Classi

import TMIDIX
from lwa_transformer import *

# %cd /content/
print('=' * 70)
print('Loading aux Classi modeules...')

import matplotlib.pyplot as plt

from torchsummary import summary
from sklearn import metrics

from midi2audio import FluidSynth
from IPython.display import Audio, display

print('=' * 70)
print('Done!')
print('Enjoy! :)')
print('=' * 70)

"""# (LOAD MODEL)"""

# Commented out IPython magic to ensure Python compatibility.
#@title Unzip Pre-Trained Classi Model
print('=' * 70)
# %cd /content/Classi/Model

print('=' * 70)
print('Unzipping pre-trained Classi model...Please wait...')

!cat /content/Classi/Model/Classi_Trained_Model.zip* > /content/Classi/Model/Classi_Trained_Model.zip
print('=' * 70)

!unzip -j /content/Classi/Model/Classi_Trained_Model.zip
print('=' * 70)

print('Done! Enjoy! :)')
print('=' * 70)
# %cd /content/
print('=' * 70)

#@title Load Classi Pre-Trained Model
full_path_to_model_checkpoint = "/content/Classi/Model/Classi_Trained_Model_33307_steps_1.1615_loss.pth" #@param {type:"string"}

print('=' * 70)
print('Loading Classi Pre-Trained Model...')
print('Please wait...')
print('=' * 70)
print('Instantiating model...')

SEQ_LEN = 2048

# instantiate the model

model = LocalTransformer(
    num_tokens = 2768,
    dim = 512,
    depth = 32,
    causal = True,
    local_attn_window_size = 512,
    max_seq_len = SEQ_LEN
).cuda()
print('=' * 70)

print('Loading model checkpoint...')

model.load_state_dict(torch.load(full_path_to_model_checkpoint))
print('=' * 70)

model.eval()

print('Done!')
print('=' * 70)

# Model stats
print('Model summary...')
summary(model)

# Plot Token Embeddings
tok_emb = model.token_emb.weight.detach().cpu().tolist()

tok_emb1 = []

for t in tok_emb:
    tok_emb1.append([abs(statistics.median(t))])

cos_sim = metrics.pairwise_distances(
   tok_emb1, metric='euclidean'
)
plt.figure(figsize=(7, 7))
plt.imshow(cos_sim, cmap="inferno", interpolation="nearest")
im_ratio = cos_sim.shape[0] / cos_sim.shape[1]
plt.colorbar(fraction=0.046 * im_ratio, pad=0.04)
plt.xlabel("Position")
plt.ylabel("Position")
plt.tight_layout()
plt.plot()
plt.savefig("/content/Classi-Tokens-Embeddings-Plot.png", bbox_inches="tight")

#@title Load Classi Artists-Pieces Dictionary
artists_pieces_dict = TMIDIX.Tegridy_Any_Pickle_File_Reader('/content/Classi/Model/Classi_Artists_Pieces_Dict')
print('Done!')

"""# (LOAD SEED MIDI)"""

#@title Load Seed MIDI
full_path_to_custom_seed_MIDI = "/content/Classi/Nothing Else Matters.kar" #@param {type:"string"}

f = full_path_to_custom_seed_MIDI

print('=' * 70)
print('Classi Seed MIDI Loader')
print('=' * 70)
print('Loading seed MIDI...')
print('=' * 70)
print('File:', f)
print('=' * 70)

#============================================================

# Helper functions

#============================================================
#============================================================

def index2tokens(index, range=32, shift=0):

  t1 = (index // range) // range
  t2 = (index // range) % range
  t3 = index % range

  return [t1 + shift, t2 + shift, t3 + shift]

def tokens2index(tokens_list, range=32, shift=0):

  t1 = tokens_list[0] - shift
  t2 = tokens_list[1] - shift
  t3 = tokens_list[2] - shift

  return (t1 * range * range) + (t2 * range) + t3

#===========================================================

melody_chords_f = []

stats = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

total_chunks_count = 0

print('Processing MIDI files. Please wait...')
print('=' * 70)

'''
fn = os.path.basename(f)
fn1 = fn.split('.mid')[0]

art = fn1.split(' --- ')[1]
pie = fn1.split(' --- ')[0]

artist1 = art.lower().strip().title()
piece1 = pie.lower().strip().title()

artist2= artist1.replace('-', ' ').replace('_', ' ').replace('.', ' ')
piece2 = piece1.replace('-', ' ').replace('_', ' ').replace('.', ' ')

artist3 = ''.join([ch[0] for ch in list(artist2) if ch[0].isalpha() or ch[0] == chr(32)])
piece3 = ''.join([ch[0] for ch in list(piece2)if ch[0].isalpha() or ch[0] == chr(32)])

artist = ' '.join([word.strip() for word in artist3.strip().split()])
piece = ' '.join([word.strip() for word in piece3.strip().split()])

# Filtering out giant MIDIs
file_size = os.path.getsize(f)
'''
#=======================================================
# START PROCESSING

# Convering MIDI to ms score with MIDI.py module
score = TMIDIX.midi2ms_score(open(f, 'rb').read())

# INSTRUMENTS CONVERSION CYCLE
events_matrix = []
itrack = 1
patches = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

patch_map = [[0, 1, 2, 3, 4, 5, 6, 7], # Piano 
          [24, 25, 26, 27, 28, 29, 30], # Guitar
          [32, 33, 34, 35, 36, 37, 38, 39], # Bass
          [40, 41], # Violin
          [42, 43], # Cello
          [46], # Harp
          [56, 57, 58, 59, 60], # Trumpet
          [71, 72], # Clarinet
          [73, 74, 75], # Flute
          [-1], # Drums
          [52, 53], # Choir
          [16, 17, 18, 19, 20] # Organ
          ]

while itrack < len(score):
  for event in score[itrack]:         
      if event[0] == 'note' or event[0] == 'patch_change':
          events_matrix.append(event)
  itrack += 1

events_matrix.sort(key=lambda x: x[1])

events_matrix1 = []

for event in events_matrix:
      if event[0] == 'patch_change':
          patches[event[2]] = event[3]

      if event[0] == 'note':
          event.extend([patches[event[3]]])
          once = False

          for p in patch_map:
              if event[6] in p and event[3] != 9: # Except the drums
                  event[3] = patch_map.index(p)
                  once = True

          if not once and event[3] != 9: # Except the drums
              event[3] = 15 # All other instruments/patches channel
              event[5] = max(80, event[5])

          if event[3] < 12: # We won't write chans 12-16 for now...
              events_matrix1.append(event)
              stats[event[3]] += 1

#=======================================================
# PRE-PROCESSING

# checking number of instruments in a composition
instruments_list_without_drums = list(set([y[3] for y in events_matrix1 if y[3] != 9]))

if len(events_matrix1) > 0 and len(instruments_list_without_drums) > 0:

    # recalculating timings
    for e in events_matrix1:
        e[1] = math.ceil(e[1] / 8) # Max 1 seconds for start-times
        e[2] = math.ceil(e[2] / 16) # Max 2 seconds for durations

    # Sorting by pitch, then by start-time
    events_matrix1.sort(key=lambda x: x[4], reverse=True)
    events_matrix1.sort(key=lambda x: x[1])

    #=======================================================
    # FINAL PRE-PROCESSING

    melody_chords = []

    pe = events_matrix1[0]

    for e in events_matrix1:
      if e[1] >= 0 and e[2] > 0:

        # Cliping all values...
        time = max(0, min(127, e[1]-pe[1]))             
        dur = max(1, min(127, e[2]))
        cha = max(0, min(11, e[3]))
        ptc = max(1, min(127, e[4]))
        vel = max(8, min(127, e[5]))

        velocity = round(vel / 15)

        # Writing a note 
        melody_chords.append([time, dur, cha, ptc, velocity])

        pe = e

    # Filtering out bad/udesirable MIDIs...
    if len([y for y in melody_chords if y[2] != 9]) > 12: 

      times = [y[0] for y in melody_chords]
      avg_time = sum(times) / len(times)

      if avg_time < 112: # Filtering out bad/udesirable MIDIs...
        if list(set([y[2] for y in melody_chords])) != [9]:
            if len(melody_chords) >= 678:

              #=======================================================
              # FINAL PROCESSING
              #=======================================================

              #=======================================================
              # MAIN PROCESSING CYCLE
              #=======================================================

              mel_chords = []

              # Classifier seq
              classifier_seq = []

              classifier_seq.extend([2688])
              artist_tokens = index2tokens(artists_pieces_dict[0].index(''), 30, 2689)
              classifier_seq.extend(artist_tokens)
              piece_tokens = index2tokens(artists_pieces_dict[1].index(''), 49, 2719)
              classifier_seq.extend(piece_tokens)

              # TOTAL DICTIONARY SIZE

              DICT_SIZE = 2768

              #=======================================================

              mel_cho = []

              comp_middle = int(len(melody_chords) / 2)

              for m in melody_chords[comp_middle-339:comp_middle+340]:

                if len(mel_cho) == (678 * 3): # 680 notes 3 tokens each

                  mel_chords = []

                  # Classified seq
                  mel_chords.extend(classifier_seq)
                  mel_chords.extend(mel_cho)
                  mel_chords.extend(classifier_seq)

                  if len(mel_chords) == 2048:
                    melody_chords_f.append(mel_chords)
                    total_chunks_count += 1

                  mel_cho = []

                # WRITING EACH NOTE HERE
                dur_vel = (m[1] * 8) + (m[4]-1)
                cha_ptc = (m[2] * 128) + m[3]

                mel_cho.extend([m[0], dur_vel+128, cha_ptc+1152])

#=======================================================
print('Done!')

#=======================================================

song = melody_chords_f[0]
song_f = []
time = 0
dur = 0
vel = 0
pitch = 0
channel = 0

son = []
song1 = []

for s in song:
  if s > 128 and s < (12*128)+1152:
    son.append(s)
  else:
    if len(son) == 3:
      song1.append(son)
    son = []
    son.append(s)
                
for ss in song1:

  time += ss[0] * 8

  dur = ((ss[1]-128) // 8) * 16
  vel = (((ss[1]-128) % 8)+1) * 15

  channel = (ss[2]-1152) // 128
  pitch = (ss[2]-1152) % 128
                  
  song_f.append(['note', time, dur, channel, pitch, vel ])

detailed_stats = TMIDIX.Tegridy_SONG_to_MIDI_Converter(song_f,
                                                    output_signature = 'Classi Classifier',  
                                                    output_file_name = '/content/Classi-Classifier-Seed-Composition', 
                                                    track_name='Project Los Angeles',
                                                    list_of_MIDI_patches=[0, 24, 32, 40, 42, 46, 56, 71, 73, 0, 53, 19, 0, 0, 0, 0],
                                                    number_of_ticks_per_quarter=500)

#=======================================================

print('=' * 70)
print('Composition stats:')
#print('Composition has', len(melody_chords_f1), 'notes')
print('Composition has', len(melody_chords_f[0]), 'tokens')
print('=' * 70)

print('Displaying resulting composition...')
print('=' * 70)

fname = '/content/Classi-Classifier-Seed-Composition'

x = []
y =[]
c = []

colors = ['red', 'yellow', 'green', 'cyan', 'blue', 'pink', 'orange', 'purple', 'gray', 'white', 'gold', 'silver']

for s in song_f:
  x.append(s[1] / 1000)
  y.append(s[4])
  c.append(colors[s[3]])

FluidSynth("/usr/share/sounds/sf2/FluidR3_GM.sf2", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))
display(Audio(str(fname + '.wav'), rate=16000))

plt.figure(figsize=(14,5))
ax=plt.axes(title=fname)
ax.set_facecolor('black')

plt.scatter(x,y, c=c)
plt.xlabel("Time")
plt.ylabel("Pitch")
plt.show()

#@title Classify
model.eval()
inp = [melody_chords_f[0][:-6]] * 4

inp = torch.LongTensor(inp).cuda()

print(inp)

sample = model.generate(inp, 6, temperature=0.8)

print(sample)

for i in range(len(sample)):
  print('=' * 70)
  sample1 = sample[i].tolist()
  print(sample1)

  artist_idx = tokens2index(sample1[0:3], 30, 2689)

  print(artist_idx)
  print(artists_pieces_dict[0][artist_idx])

  piece_idx = tokens2index(sample1[3:6], 49, 2719)

  print(piece_idx)
  print(artists_pieces_dict[1][piece_idx])

"""# Congrats! You did it :)"""