wikibert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyNIDrtAso/BtfDN6RBX3WPV",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/trbndev/wikibert/blob/main/wikibert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%pip install tensorflow"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zhSINCW6sZdS",
        "outputId": "3030c784-9bb5-4200-cc9e-110aba946001"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: tensorflow in /usr/local/lib/python3.10/dist-packages (2.17.1)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.4.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.6.3)\n",
            "Requirement already satisfied: flatbuffers>=24.3.25 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (24.3.25)\n",
            "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.6.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied: h5py>=3.10.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.12.1)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (18.1.1)\n",
            "Requirement already satisfied: ml-dtypes<0.5.0,>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.4.1)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.4.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow) (24.2)\n",
            "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (4.25.5)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.32.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow) (75.1.0)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.17.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.5.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (4.12.2)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.17.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.68.1)\n",
            "Requirement already satisfied: tensorboard<2.18,>=2.17 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (2.17.1)\n",
            "Requirement already satisfied: keras>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (3.5.0)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (0.37.1)\n",
            "Requirement already satisfied: numpy<2.0.0,>=1.23.5 in /usr/local/lib/python3.10/dist-packages (from tensorflow) (1.26.4)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow) (0.45.1)\n",
            "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow) (13.9.4)\n",
            "Requirement already satisfied: namex in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow) (0.0.8)\n",
            "Requirement already satisfied: optree in /usr/local/lib/python3.10/dist-packages (from keras>=3.2.0->tensorflow) (0.13.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorflow) (3.4.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorflow) (3.10)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorflow) (2.2.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2.21.0->tensorflow) (2024.8.30)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow) (3.7)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.18,>=2.17->tensorflow) (3.1.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.18,>=2.17->tensorflow) (3.0.2)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->keras>=3.2.0->tensorflow) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->keras>=3.2.0->tensorflow) (2.18.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->keras>=3.2.0->tensorflow) (0.1.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ixFTE3ADsRI_"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers\n",
        "import numpy as np\n",
        "import os"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the text data\n",
        "file_path = './all_articles.txt'\n",
        "text = open(file_path, 'r', encoding='utf-8').read()\n",
        "\n",
        "# Convert text to lowercase\n",
        "text = text.lower()\n",
        "\n",
        "# Create a sorted list of unique characters\n",
        "vocab = sorted(set(text))\n",
        "\n",
        "# Create mappings from characters to indices and vice versa\n",
        "char2idx = {char: idx for idx, char in enumerate(vocab)}\n",
        "idx2char = np.array(vocab)\n",
        "\n",
        "# Convert the text to a sequence of integers\n",
        "text_as_int = np.array([char2idx[char] for char in text])"
      ],
      "metadata": {
        "id": "gJT6ftdUscxw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Set the length of sequences for input\n",
        "seq_length = 100\n",
        "examples_per_epoch = len(text) // (seq_length + 1)\n",
        "\n",
        "# Create training sequences\n",
        "char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)\n",
        "\n",
        "# Group sequences\n",
        "sequences = char_dataset.batch(seq_length + 1, drop_remainder=True)\n",
        "\n",
        "# Function to split input and target\n",
        "def split_input_target(chunk):\n",
        "    input_text = chunk[:-1]\n",
        "    target_text = chunk[1:]\n",
        "    return input_text, target_text\n",
        "\n",
        "# Apply the function to sequences\n",
        "dataset = sequences.map(split_input_target)"
      ],
      "metadata": {
        "id": "UhkYWs5XsgZJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Batch size\n",
        "BATCH_SIZE = 64\n",
        "\n",
        "# Buffer size for shuffling\n",
        "BUFFER_SIZE = 10000\n",
        "\n",
        "# Create training batches\n",
        "data = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)"
      ],
      "metadata": {
        "id": "2-U7Ew3Lshp9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Length of the vocabulary\n",
        "vocab_size = len(vocab)\n",
        "\n",
        "# Embedding dimensions and RNN units\n",
        "embedding_dim = 256\n",
        "rnn_units = 1024\n",
        "\n",
        "# Build the model\n",
        "# Build the model with an Input layer\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Input(batch_shape=(BATCH_SIZE, None)),\n",
        "    tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim),\n",
        "    tf.keras.layers.GRU(units=rnn_units, return_sequences=True, stateful=True, recurrent_initializer='glorot_uniform'),\n",
        "    tf.keras.layers.Dense(units=vocab_size)\n",
        "])"
      ],
      "metadata": {
        "id": "KZ3BCgpisjUv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Custom loss function\n",
        "def loss(labels, logits):\n",
        "    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)"
      ],
      "metadata": {
        "id": "LN4JJC3Qsm-j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Compile the model\n",
        "model.compile(optimizer='adam', loss=loss)"
      ],
      "metadata": {
        "id": "FcvT6rpWsoLG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Directory to save checkpoints\n",
        "checkpoint_dir = './training_checkpoints'\n",
        "os.makedirs(checkpoint_dir, exist_ok=True)\n",
        "\n",
        "# Checkpoint file prefix\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_{epoch}.weights.h5\")\n",
        "\n",
        "# Checkpoint callback\n",
        "checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_prefix,\n",
        "    save_weights_only=True\n",
        ")"
      ],
      "metadata": {
        "id": "AoYjBXOlso-S"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Number of epochs for training\n",
        "EPOCHS = 20\n",
        "\n",
        "# Train the model\n",
        "model.fit(data, epochs=EPOCHS, callbacks=[checkpoint_callback])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-Dich5-ssqr3",
        "outputId": "f13fc1eb-fcb3-4557-e638-5e67c86b17e8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m138s\u001b[0m 70ms/step - loss: 1.9979\n",
            "Epoch 2/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m141s\u001b[0m 71ms/step - loss: 1.2927\n",
            "Epoch 3/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m136s\u001b[0m 71ms/step - loss: 1.2291\n",
            "Epoch 4/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m137s\u001b[0m 71ms/step - loss: 1.1979\n",
            "Epoch 5/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m136s\u001b[0m 71ms/step - loss: 1.1795\n",
            "Epoch 6/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m143s\u001b[0m 71ms/step - loss: 1.1686\n",
            "Epoch 7/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m137s\u001b[0m 72ms/step - loss: 1.1612\n",
            "Epoch 8/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m135s\u001b[0m 71ms/step - loss: 1.1579\n",
            "Epoch 9/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m141s\u001b[0m 71ms/step - loss: 1.1566\n",
            "Epoch 10/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m141s\u001b[0m 70ms/step - loss: 1.1589\n",
            "Epoch 11/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m134s\u001b[0m 70ms/step - loss: 1.1647\n",
            "Epoch 12/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m141s\u001b[0m 70ms/step - loss: 1.1755\n",
            "Epoch 13/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m142s\u001b[0m 70ms/step - loss: 1.2159\n",
            "Epoch 14/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m134s\u001b[0m 70ms/step - loss: 1.2710\n",
            "Epoch 15/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m141s\u001b[0m 70ms/step - loss: 1.4598\n",
            "Epoch 16/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m134s\u001b[0m 70ms/step - loss: 1.8718\n",
            "Epoch 17/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m140s\u001b[0m 70ms/step - loss: 1.9520\n",
            "Epoch 18/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m141s\u001b[0m 69ms/step - loss: 2.0282\n",
            "Epoch 19/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m142s\u001b[0m 69ms/step - loss: 1.9805\n",
            "Epoch 20/20\n",
            "\u001b[1m1879/1879\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m143s\u001b[0m 70ms/step - loss: 1.9588\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.callbacks.history.History at 0x7d17bc30b3d0>"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Rebuild the model with batch size of 1 for text generation\n",
        "model = tf.keras.Sequential([\n",
        "    tf.keras.layers.Input(batch_shape=[1, None]),\n",
        "    layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim),\n",
        "    layers.GRU(\n",
        "        units=rnn_units,\n",
        "        return_sequences=True,\n",
        "        stateful=True,\n",
        "        recurrent_initializer='glorot_uniform'\n",
        "    ),\n",
        "    layers.Dense(units=vocab_size)\n",
        "])\n",
        "\n",
        "# Load the trained weights\n",
        "best_checkpoint = os.path.join(checkpoint_dir, \"ckpt_9.weights.h5\")\n",
        "\n",
        "model.load_weights(best_checkpoint)\n",
        "model.build(tf.TensorShape([1, None]))"
      ],
      "metadata": {
        "id": "hYpaB_uistnY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_text(model, start_string, num_generate=1000):\n",
        "    start_string = start_string.lower()\n",
        "    # Convert the start string to numbers (vectorize)\n",
        "    input_eval = [char2idx[char] for char in start_string]\n",
        "    input_eval = tf.expand_dims(input_eval, 0)\n",
        "\n",
        "    # Empty list to store generated text\n",
        "    text_generated = []\n",
        "\n",
        "    # Temperature for prediction diversity\n",
        "    temperature = 0.7\n",
        "\n",
        "    # Reset the states of the RNN layers\n",
        "    # If you have multiple RNN layers, reset each one\n",
        "    for layer in model.layers:\n",
        "        if hasattr(layer, 'reset_states'):\n",
        "            layer.reset_states()\n",
        "\n",
        "    # Generate text\n",
        "    for _ in range(num_generate):\n",
        "        predictions = model(input_eval)\n",
        "        predictions = tf.squeeze(predictions, 0)\n",
        "\n",
        "        # Apply temperature\n",
        "        predictions = predictions / temperature\n",
        "\n",
        "        # Predicted ID is sampled from the probability distribution\n",
        "        predicted_id = tf.random.categorical(predictions, num_samples=1)[-1, 0].numpy()\n",
        "\n",
        "        # Pass the predicted ID as the next input\n",
        "        input_eval = tf.expand_dims([predicted_id], 0)\n",
        "\n",
        "        # Append the predicted character to the generated text\n",
        "        text_generated.append(idx2char[predicted_id])\n",
        "\n",
        "    return start_string + ''.join(text_generated)"
      ],
      "metadata": {
        "id": "kLpPxuoisvz6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Start string for text generation\n",
        "start_string = \"Windows 8 was\"\n",
        "\n",
        "# Generate and print the text\n",
        "generated_text = generate_text(model, start_string=start_string)\n",
        "print(generated_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K-tRbp56sxG_",
        "outputId": "0ec097c2-9254-452b-93be-6af7228d2cfd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "windows 8 was released in 1987 for development environments. it was introduced to note that the second alphabet is the oldest proceeds of ram in october 2007 with all of the sphere with a string list in common oriented language storage of a polyhedron is the form of syntax. since the result is the second schipping, a second sequence are typically available to a solaris or by control structures of a special line, such as a legal field by internet architecture and microsoft both installations, which are also an implementation of many other available versions of windows powers with updates to be ported to a single studio tables.\n",
            "\n",
            "numerical and events and skills to the nature of a scaling a program within the x86 architecture of the test will be installed free safety for optional techniques used in components were also available to default variants or the program ased the classification of the fact that the first purpose of windows vista is also used as a security update to allow the program must be up\n"
          ]
        }
      ]
    }
  ]
}