notebooks/AI_Music_Generator_LSTM_CNN_GAN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Libraries imported successfully!\n"
     ]
    }
   ],
   "source": [
    "# Importing necessary libraries\n",
    "import os\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input, LSTM, Dense, Conv2D, MaxPooling2D, Flatten, Dropout\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import librosa\n",
    "import pickle\n",
    "from skimage.transform import resize\n",
    "\n",
    "print(\"Libraries imported successfully!\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding functions...\n",
      "Functions added successfully!\n"
     ]
    }
   ],
   "source": [
    "print(\"Adding functions...\")\n",
    "\n",
    "\n",
    "# Function to walk through a directory and collect WAV files\n",
    "def collect_audio_files(directory):\n",
    "    audio_files = []\n",
    "    for root, dirs, files in os.walk(directory):\n",
    "        for file in files:\n",
    "            if file.endswith('.wav'):\n",
    "                audio_files.append(os.path.join(root, file))\n",
    "    return audio_files\n",
    "\n",
    "\n",
    "# Modify the preprocess_and_store_data function to use resize_spectrogram\n",
    "def preprocess_and_store_data(audio_files, feature_store_path):\n",
    "    spectrograms = []\n",
    "    target_shape = (128, 128)  # Desired shape for CNN input\n",
    "\n",
    "    for file in audio_files:\n",
    "        audio, sr = librosa.load(file, sr=None)\n",
    "        spectrogram = librosa.feature.melspectrogram(audio, sr=sr)\n",
    "        spectrogram = librosa.power_to_db(spectrogram, ref=np.max)\n",
    "        spectrogram = (spectrogram - np.mean(spectrogram)) / np.std(spectrogram)\n",
    "\n",
    "        # Resize spectrogram to target_shape (128x128)\n",
    "        spectrogram = resize_spectrogram(spectrogram, target_shape)\n",
    "\n",
    "        # Add a channel dimension\n",
    "        spectrogram = spectrogram[..., np.newaxis]\n",
    "\n",
    "        spectrograms.append(spectrogram)\n",
    "\n",
    "    # Store the processed data\n",
    "    with open(feature_store_path, 'wb') as f:\n",
    "        pickle.dump(spectrograms, f)\n",
    "\n",
    "    return np.array(spectrograms)\n",
    "\n",
    "\n",
    "def resize_spectrogram(spectrogram, target_shape):\n",
    "    \"\"\"\n",
    "    Resize the spectrogram to the target shape.\n",
    "    This function will crop or pad the spectrogram as necessary.\n",
    "\n",
    "    :param spectrogram: 2D array of the spectrogram\n",
    "    :param target_shape: Tuple (height, width) for the target shape\n",
    "    :return: Resized spectrogram\n",
    "    \"\"\"\n",
    "    current_shape = spectrogram.shape\n",
    "\n",
    "    # Padding if necessary\n",
    "    if current_shape[0] < target_shape[0] or current_shape[1] < target_shape[1]:\n",
    "        padding = [(0, max(0, target_shape[0] - current_shape[0])),\n",
    "                   (0, max(0, target_shape[1] - current_shape[1]))]\n",
    "        spectrogram = np.pad(spectrogram, padding, mode='constant', constant_values=0)\n",
    "\n",
    "    # Cropping if necessary\n",
    "    cropped_spectrogram = spectrogram[:target_shape[0], :target_shape[1]]\n",
    "\n",
    "    return cropped_spectrogram\n",
    "\n",
    "\n",
    "# Load preprocessed data from storage\n",
    "def load_preprocessed_data(feature_store_path):\n",
    "    with open(feature_store_path, 'rb') as f:\n",
    "        spectrograms = pickle.load(f)\n",
    "    return np.array(spectrograms)\n",
    "\n",
    "\n",
    "# CNN for Feature Extraction\n",
    "def create_cnn():\n",
    "    input_shape = (128, 128, 1)\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = Conv2D(32, (3, 3), activation='relu')(inputs)\n",
    "    x = MaxPooling2D((2, 2))(x)\n",
    "    x = Conv2D(64, (3, 3), activation='relu')(x)\n",
    "    x = MaxPooling2D((2, 2))(x)\n",
    "    x = Flatten()(x)\n",
    "    model = Model(inputs, x)\n",
    "    return model\n",
    "\n",
    "# LSTM for Temporal Dynamics\n",
    "def create_lstm(cnn_output_size):\n",
    "    inputs = Input(shape=(None, cnn_output_size))\n",
    "    x = LSTM(256, return_sequences=True)(inputs)\n",
    "    x = Dropout(0.3)(x)\n",
    "    x = LSTM(256)(x)\n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    model = Model(inputs, x)\n",
    "    return model\n",
    "\n",
    "# GAN for Music Generation\n",
    "def create_gan(cnn, lstm):\n",
    "    generator = lstm\n",
    "    discriminator = cnn\n",
    "    discriminator.compile(loss='binary_crossentropy', optimizer=Adam(0.0002, 0.5))\n",
    "    z = Input(shape=(None, cnn.output_shape[-1]))\n",
    "    music = generator(z)\n",
    "    discriminator.trainable = False\n",
    "    validity = discriminator(music)\n",
    "    combined = Model(z, validity)\n",
    "    combined.compile(loss='binary_crossentropy', optimizer=Adam(0.0002, 0.5))\n",
    "    return combined\n",
    "\n",
    "# Main Training Loop\n",
    "def train_gan(combined, generator, discriminator, spectrograms, epochs, batch_size):\n",
    "    for epoch in range(epochs):\n",
    "        idx = np.random.randint(0, spectrograms.shape[0], batch_size)\n",
    "        real_spectrograms = spectrograms[idx]\n",
    "        noise = np.random.normal(0, 1, (batch_size, 100))\n",
    "        generated_spectrograms = generator.predict(noise)\n",
    "        real_loss = discriminator.train_on_batch(real_spectrograms, np.ones((batch_size, 1)))\n",
    "        fake_loss = discriminator.train_on_batch(generated_spectrograms, np.zeros((batch_size, 1)))\n",
    "        discriminator_loss = 0.5 * np.add(real_loss, fake_loss)\n",
    "        noise = np.random.normal(0, 1, (batch_size, 100))\n",
    "        generator_loss = combined.train_on_batch(noise, np.ones((batch_size, 1)))\n",
    "        print(f\"Epoch {epoch} / {epochs} - Discriminator Loss: {discriminator_loss}, Generator Loss: {generator_loss}\")\n",
    "\n",
    "# Music Generation Function\n",
    "def generate_music(generator, num_samples):\n",
    "    noise = np.random.normal(0, 1, (num_samples, 100))\n",
    "    generated_spectrograms = generator.predict(noise)\n",
    "    return generated_spectrograms\n",
    "\n",
    "\n",
    "# AWS Integration Placeholder (Dormant)\n",
    "def aws_integration_placeholder():\n",
    "    pass\n",
    "\n",
    "print(\"Functions added successfully!\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unexpected exception formatting exception. Falling back to standard exception\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3548, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"C:\\Users\\DedHawk\\AppData\\Local\\Temp\\ipykernel_2292\\3485226287.py\", line 15, in <module>\n",
      "    spectrograms = preprocess_and_store_data(audio_files, feature_store_path)\n",
      "  File \"C:\\Users\\DedHawk\\AppData\\Local\\Temp\\ipykernel_2292\\3650180692.py\", line 20, in preprocess_and_store_data\n",
      "    audio, sr = librosa.load(file, sr=None)\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\lazy_loader\\__init__.py\", line 78, in __getattr__\n",
      "    attr = getattr(submod, name)\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\lazy_loader\\__init__.py\", line 77, in __getattr__\n",
      "    submod = importlib.import_module(submod_path)\n",
      "  File \"C:\\Users\\DedHawk\\AppData\\Local\\Programs\\Python\\Python39\\lib\\importlib\\__init__.py\", line 127, in import_module\n",
      "    return _bootstrap._gcd_import(name[level:], package, level)\n",
      "  File \"<frozen importlib._bootstrap>\", line 1030, in _gcd_import\n",
      "  File \"<frozen importlib._bootstrap>\", line 1007, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 986, in _find_and_load_unlocked\n",
      "  File \"<frozen importlib._bootstrap>\", line 680, in _load_unlocked\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 850, in exec_module\n",
      "  File \"<frozen importlib._bootstrap>\", line 228, in _call_with_frames_removed\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\librosa\\core\\audio.py\", line 13, in <module>\n",
      "    import scipy.signal\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\scipy\\signal\\__init__.py\", line 333, in <module>\n",
      "    from ._peak_finding import *\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\scipy\\signal\\_peak_finding.py\", line 8, in <module>\n",
      "    from scipy.stats import scoreatpercentile\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\scipy\\stats\\__init__.py\", line 608, in <module>\n",
      "    from ._stats_py import *\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\scipy\\stats\\_stats_py.py\", line 41, in <module>\n",
      "    from scipy._lib._util import (check_random_state, MapWrapper, _get_nan,\n",
      "ImportError: cannot import name '_get_nan' from 'scipy._lib._util' (C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\scipy\\_lib\\_util.py)\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2142, in showtraceback\n",
      "    stb = self.InteractiveTB.structured_traceback(\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 1435, in structured_traceback\n",
      "    return FormattedTB.structured_traceback(\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 1326, in structured_traceback\n",
      "    return VerboseTB.structured_traceback(\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 1173, in structured_traceback\n",
      "    formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 1063, in format_exception_as_a_whole\n",
      "    self.get_records(etb, number_of_lines_of_context, tb_offset) if etb else []\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 1155, in get_records\n",
      "    FrameInfo(\n",
      "  File \"C:\\Users\\DedHawk\\PycharmProjects\\dedaiInstrumentAI\\JUCE\\modules\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 780, in __init__\n",
      "    ix = inspect.getsourcelines(frame)\n",
      "  File \"C:\\Users\\DedHawk\\AppData\\Local\\Programs\\Python\\Python39\\lib\\inspect.py\", line 1006, in getsourcelines\n",
      "    lines, lnum = findsource(object)\n",
      "  File \"C:\\Users\\DedHawk\\AppData\\Local\\Programs\\Python\\Python39\\lib\\inspect.py\", line 835, in findsource\n",
      "    raise OSError('could not get source code')\n",
      "OSError: could not get source code\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Directory containing WAV files\n",
    "directory = 'datasets/nsynth/audio'\n",
    "audio_files = collect_audio_files(directory)\n",
    "\n",
    "# Path to store extracted features\n",
    "feature_store_path = 'datasets/nsynth/store.pkl'\n",
    "\n",
    "# Ensure the directory for store.pkl exists\n",
    "os.makedirs(os.path.dirname(feature_store_path), exist_ok=True)\n",
    "\n",
    "# Check if preprocessed data exists\n",
    "if os.path.exists(feature_store_path):\n",
    "    spectrograms = load_preprocessed_data(feature_store_path)\n",
    "else:\n",
    "    spectrograms = preprocess_and_store_data(audio_files, feature_store_path)\n",
    "\n",
    "# Prepare models\n",
    "cnn = create_cnn()\n",
    "lstm = create_lstm(cnn.output_shape[-1])\n",
    "gan = create_gan(cnn, lstm)\n",
    "\n",
    "# Train the GAN\n",
    "train_gan(gan, lstm, cnn, spectrograms, epochs=1000, batch_size=32)\n",
    "\n",
    "# Generate music\n",
    "generated_music = generate_music(lstm, 5)\n",
    "# Additional processing to convert spectrograms back to audio"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}