{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "frame-classifier-final.ipynb", "provenance": [], "collapsed_sections": [], "toc_visible": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "jVZj7qrM1EdH", "colab_type": "text" }, "source": [ "### Frame Classifier Final\n", "\n", "This notebook creates the final frame classification using the best model we have observed:\n", "\n", "- ResNet with LSTM (x3)" ] }, { "cell_type": "code", "metadata": { "id": "C6Ynw1iGiNqT", "colab_type": "code", "colab": {} }, "source": [ "import numpy as np\n", "import random\n", "import cv2\n", "from os import listdir\n", "from os.path import isfile, join\n", "import tensorflow as tf\n", "\n", "class FramesDataGenerator(tf.keras.utils.Sequence):\n", " \"\"\"\n", " Custom Keras generator for frames\n", " \"\"\"\n", " def __init__(self, dir, batch_size=32, frames_to_use=12, is_test=False, shuffle=True, height=320, width=480):\n", " self.frames_to_use = frames_to_use\n", " self.height = height\n", " self.width = width\n", " self.batch_size = batch_size\n", " self.is_test = is_test\n", " self.dir = dir\n", " self.shuffle = shuffle\n", " self.classes = self.find_classes()\n", " self.video_names, self.video_map, self.video_to_class, self.num_samples, self.min_frames = self.find_samples()\n", " self.on_epoch_end()\n", " if self.is_test:\n", " print(f\"Found {self.num_samples} frames belonging to {len(self.video_names)} videos (test-mode).\")\n", " else:\n", " print(f\"Found {self.num_samples} frames belonging to {len(self.video_names)} videos belonging to {len(self.classes)} classes.\")\n", " print(f\"Min frames determined to be {self.min_frames}\")\n", "\n", " def find_classes(self):\n", " if self.is_test:\n", " return []\n", " else:\n", " category_folders = [f for f in listdir(self.dir) if not isfile(join(self.dir, f))]\n", " return sorted(list(set(category_folders)))\n", "\n", " def find_samples(self):\n", " num_samples = 0\n", " min_frames = -1\n", " video_map = {}\n", " vid_to_cat = {}\n", " if self.is_test:\n", " category_folders = [self.dir]\n", " else:\n", " category_folders = [f for f in listdir(self.dir) if not isfile(join(self.dir, f))]\n", " for category_folder in category_folders:\n", " if self.is_test:\n", " cat_path = category_folder\n", " else:\n", " cat_path = join(self.dir, category_folder)\n", " frames = [f for f in listdir(cat_path) if isfile(join(cat_path, f))]\n", " for frame in frames:\n", " # frame = frame_101_7.mp4_8.jpg\n", " frame_arr = frame.split(\".mp4_\")\n", " vid_name = frame_arr[0]\n", " if vid_name not in video_map:\n", " video_map[vid_name] = []\n", " vid_to_cat[vid_name] = category_folder\n", " video_map[vid_name].append(frame)\n", " \n", " for k in video_map.keys():\n", " # make sure the frames for each video are in sorted order\n", " video_map[vid_name] = sorted(video_map[vid_name])\n", " if min_frames == -1 or len(video_map[vid_name]) < min_frames:\n", " min_frames = len(video_map[vid_name])\n", "\n", " return list(video_map.keys()), video_map, vid_to_cat, len(vid_to_cat), min_frames\n", "\n", " def __len__(self):\n", " \"\"\"\n", " Denotes the number of batches per epoch\n", " \"\"\"\n", " return int(np.floor(self.num_samples / self.batch_size))\n", "\n", " def __getitem__(self, index):\n", " \"\"\"\n", " Generate one batch of data\n", " \"\"\"\n", " video_names = self.video_names[index*self.batch_size:(index+1)*self.batch_size]\n", " num_frames = self.min_frames if self.frames_to_use == -1 else self.frames_to_use\n", " X = np.zeros((len(video_names), num_frames, self.height, self.width, 3), dtype=np.uint8)\n", " y = []\n", " i = 0\n", " for vid in video_names:\n", " j = 0\n", " for frame in self.video_map[vid]:\n", " if self.is_test:\n", " frame_path = join(self.dir, frame)\n", " else:\n", " frame_path = join(join(self.dir, self.video_to_class[vid]), frame)\n", "\n", " img = cv2.imread(frame_path)\n", " img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n", " X[i, j, :, :, :] = img\n", " j += 1\n", " if j >= num_frames:\n", " break\n", "\n", " if self.is_test:\n", " y.append(0)\n", " else:\n", " y.append(int(self.video_to_class[vid]) - 1)\n", " i += 1\n", " y = np.array(y)\n", " return X, tf.keras.utils.to_categorical(y, num_classes=len(self.classes))\n", "\n", " def on_epoch_end(self):\n", " if self.shuffle == True:\n", " np.random.shuffle(self.video_names)\n" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "Fpl8T7NAqeNB", "colab_type": "code", "colab": {} }, "source": [ "import zipfile\n", "import cv2\n", "from concurrent.futures import ThreadPoolExecutor\n", "from pathlib import Path\n", "import os\n", "from os.path import isfile, join\n", "import tensorflow as tf\n", "import tempfile\n", "import shutil\n", "\n", "\n", "class FramesClassifier:\n", " \"\"\"\n", " Classifies sentiment based on frames extracted from video clips\n", " \"\"\"\n", "\n", " def __init__(self, frames_folder, model_location=None, is_test=None, is_zip=True, frames_to_use=12, batch_size=16):\n", " \"\"\"\n", " @param frames_folder The folder where the list of frames are stored. If \n", " `is_zip` is set to True, this should be a single zip \n", " file containing the frames. Paths can either by a local \n", " folder or a GDrive mounted path.\n", " @param model_location The pre-trained model to perform predictions\n", " @param is_test If set to True, we assume that `frames_folder` contains a flat\n", " list of videos. If False, we assume that `frames_folder` first \n", " contains subdirectories corresponding to category labels. \n", " @param is_zip If set to True, the `frames_folder` will be unzipped prior to accessing\n", " @param frames_to_use The number of frames to use per video\n", " @param batch_size The batch size used to feed into the model evaluation\n", " \"\"\"\n", " self.is_zip = is_zip\n", " self.frames_folder = frames_folder\n", " self.is_test = is_test\n", " self.model_location = model_location\n", " self.frames_to_use = frames_to_use\n", " self.batch_size = batch_size\n", " print(f\"FramesClassifier created with is_zip = {is_zip}, frames_folder = {frames_folder} , is_test = {is_test} , model_location = {model_location}\")\n", "\n", " def predict(self, layer=None):\n", " folder = self.unzip_folder()\n", " generator = FramesDataGenerator(folder, is_test=self.is_test, frames_to_use=self.frames_to_use, batch_size=self.batch_size)\n", "\n", " if \"https://\" in self.model_location or \"http://\" in self.model_location:\n", " downloaded_model_path = tf.keras.utils.get_file(\"frame-classifier\", self.model_location)\n", " model = tf.keras.models.load_model(downloaded_model_path)\n", " else:\n", " model = tf.keras.models.load_model(self.model_location)\n", " if layer is not None:\n", " print(f\"Customizing model by returning layer {layer}\")\n", " model = tf.keras.models.Model(model.input, model.get_layer(layer).output)\n", " return model.predict(generator)\n", "\n", " def summary(self):\n", " model = tf.keras.models.load_model(self.model_location)\n", " model.summary()\n", "\n", " def evaluate(self):\n", " if self.is_test:\n", " print(\"Evaluation cannot be done in test-mode\")\n", " return\n", "\n", " folder = self.unzip_folder()\n", " generator = FramesDataGenerator(folder, is_test=self.is_test, frames_to_use=self.frames_to_use, batch_size=self.batch_size)\n", " model = tf.keras.models.load_model(self.model_location)\n", " return model.evaluate(generator)\n", "\n", " def unzip_folder(self):\n", " if self.is_zip:\n", " # Unzips files to a temp directory\n", " tmp_output_folder = \"frames_tmp\"\n", " if os.path.exists(tmp_output_folder) and os.path.isdir(tmp_output_folder):\n", " print(\"Removing existing dir...\")\n", " shutil.rmtree(tmp_output_folder)\n", "\n", " print(f\"Unzipping files to temp dir {tmp_output_folder}...\")\n", " Path(f\"{tmp_output_folder}\").mkdir(parents=True, exist_ok=True)\n", " with zipfile.ZipFile(self.frames_folder, 'r') as zip_ref:\n", " zip_ref.extractall(tmp_output_folder)\n", " print(\"Finished unzipping files\")\n", " else:\n", " tmp_output_folder = self.frames_folder\n", " print(\"Skipping unzipping files as input is a folder\")\n", " return tmp_output_folder\n", "\n", "\n" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "GxshK3rYJFcX", "colab_type": "code", "outputId": "e7e4a39c-5c0b-4cff-bed0-56e8c7bf742a", "colab": { "base_uri": "https://localhost:8080/", "height": 54 } }, "source": [ "from google.colab import drive\n", "drive.mount('/content/drive')\n", "\n" ], "execution_count": 36, "outputs": [ { "output_type": "stream", "text": [ "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "3dVNPRhRolot", "colab_type": "code", "colab": { "base_uri": "https://localhost:8080/", "height": 581 }, "outputId": "ff73c8c6-a5f1-4da5-ffbc-86dbc7c26522" }, "source": [ "HOME_DIR = \"drive/My Drive/\"\n", "frames_classifier = FramesClassifier(\n", " frames_folder = HOME_DIR + \"cs231n-project/datasets/emotiw/train-tiny-local.zip\", \n", " is_test = False,\n", " model_location = HOME_DIR + \"cs231n-project/models/frame-classifier-resnet-lstm-x3.h5\"\n", ")\n", "frames_classifier.summary()" ], "execution_count": 37, "outputs": [ { "output_type": "stream", "text": [ "FramesClassifier created with is_zip = True, frames_folder = drive/My Drive/cs231n-project/datasets/emotiw/train-tiny-local.zip , is_test = False , model_location = drive/My Drive/cs231n-project/models/frame-classifier-resnet-lstm-x3.h5\n", "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n", "Model: \"model_1\"\n", "__________________________________________________________________________________________________\n", "Layer (type) Output Shape Param # Connected to \n", "==================================================================================================\n", "input_4 (InputLayer) [(None, 12, 320, 480 0 \n", "__________________________________________________________________________________________________\n", "time_distributed_1 (TimeDistrib (None, 12, 10, 15, 2 23587712 input_4[0][0] \n", "__________________________________________________________________________________________________\n", "conv_lst_m2d_3 (ConvLSTM2D) (None, 12, 10, 15, 4 3006880 time_distributed_1[0][0] \n", "__________________________________________________________________________________________________\n", "conv_lst_m2d_4 (ConvLSTM2D) (None, 12, 10, 15, 4 3006880 time_distributed_1[0][0] \n", "__________________________________________________________________________________________________\n", "conv_lst_m2d_5 (ConvLSTM2D) (None, 12, 10, 15, 4 3006880 time_distributed_1[0][0] \n", "__________________________________________________________________________________________________\n", "global_average_pooling3d_3 (Glo (None, 40) 0 conv_lst_m2d_3[0][0] \n", "__________________________________________________________________________________________________\n", "global_average_pooling3d_4 (Glo (None, 40) 0 conv_lst_m2d_4[0][0] \n", "__________________________________________________________________________________________________\n", "global_average_pooling3d_5 (Glo (None, 40) 0 conv_lst_m2d_5[0][0] \n", "__________________________________________________________________________________________________\n", "concatenate_1 (Concatenate) (None, 120) 0 global_average_pooling3d_3[0][0] \n", " global_average_pooling3d_4[0][0] \n", " global_average_pooling3d_5[0][0] \n", "__________________________________________________________________________________________________\n", "dense_1 (Dense) (None, 3) 363 concatenate_1[0][0] \n", "==================================================================================================\n", "Total params: 32,608,715\n", "Trainable params: 9,021,003\n", "Non-trainable params: 23,587,712\n", "__________________________________________________________________________________________________\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "Qbh349UdM0uU", "colab_type": "code", "outputId": "5a7cd358-f9e7-4dcb-eb30-6743c0977e93", "colab": { "base_uri": "https://localhost:8080/", "height": 190 } }, "source": [ "HOME_DIR = \"drive/My Drive/\"\n", "frames_classifier = FramesClassifier(\n", " frames_folder = HOME_DIR + \"cs231n-project/datasets/emotiw/train-tiny-local.zip\", \n", " is_test = False,\n", " model_location = HOME_DIR + \"cs231n-project/models/frame-classifier-resnet-lstm-x3.h5\"\n", ")\n", "frames_classifier.evaluate()" ], "execution_count": 51, "outputs": [ { "output_type": "stream", "text": [ "FramesClassifier created with is_zip = True, frames_folder = drive/My Drive/cs231n-project/datasets/emotiw/train-tiny-local.zip , is_test = False , model_location = drive/My Drive/cs231n-project/models/frame-classifier-resnet-lstm-x3.h5\n", "Removing existing dir...\n", "Unzipping files to temp dir frames_tmp...\n", "Finished unzipping files\n", "Found 50 frames belonging to 50 videos belonging to 3 classes.\n", "Min frames determined to be 13\n", "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n", "3/3 [==============================] - 8s 3s/step - loss: 0.8295 - accuracy: 0.7083\n" ], "name": "stdout" }, { "output_type": "execute_result", "data": { "text/plain": [ "[0.8294854760169983, 0.7083333134651184]" ] }, "metadata": { "tags": [] }, "execution_count": 51 } ] }, { "cell_type": "code", "metadata": { "id": "djHyqsMl7mK6", "colab_type": "code", "outputId": "4c798223-c7b5-4ed6-a619-c42e9e9b9c80", "colab": { "base_uri": "https://localhost:8080/", "height": 1000 } }, "source": [ "HOME_DIR = \"drive/My Drive/\"\n", "frames_classifier = FramesClassifier(\n", " frames_folder = HOME_DIR + \"cs231n-project/datasets/emotiw/test-tiny-local.zip\", \n", " is_test = True,\n", " model_location = \"https://storage.googleapis.com/cs231n-emotiw/frame-classifier-resnet-lstm-x3.h5\"\n", ")\n", "frames_classifier.predict()" ], "execution_count": 52, "outputs": [ { "output_type": "stream", "text": [ "FramesClassifier created with is_zip = True, frames_folder = drive/My Drive/cs231n-project/datasets/emotiw/test-tiny-local.zip , is_test = True , model_location = https://storage.googleapis.com/cs231n-emotiw/frame-classifier-resnet-lstm-x3.h5\n", "Removing existing dir...\n", "Unzipping files to temp dir frames_tmp...\n", "Finished unzipping files\n", "Found 50 frames belonging to 50 videos (test-mode).\n", "Min frames determined to be 13\n", "Downloading data from https://storage.googleapis.com/cs231n-emotiw/frame-classifier-resnet-lstm-x3.h5\n", "202964992/202962720 [==============================] - 2s 0us/step\n", "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n" ], "name": "stdout" }, { "output_type": "execute_result", "data": { "text/plain": [ "array([[0.34820428, 0.54439634, 0.10739935],\n", " [0.2909088 , 0.10773491, 0.6013563 ],\n", " [0.54589367, 0.21219504, 0.24191126],\n", " [0.37305444, 0.45109972, 0.17584579],\n", " [0.22389376, 0.11809638, 0.6580098 ],\n", " [0.44518152, 0.47764522, 0.07717329],\n", " [0.39797345, 0.45777172, 0.14425491],\n", " [0.22224568, 0.10284888, 0.6749054 ],\n", " [0.61116457, 0.17026111, 0.21857433],\n", " [0.5346621 , 0.41566825, 0.04966953],\n", " [0.53102344, 0.19083473, 0.27814186],\n", " [0.28158262, 0.64193964, 0.07647772],\n", " [0.3075645 , 0.271059 , 0.4213765 ],\n", " [0.23010482, 0.28360716, 0.48628804],\n", " [0.2536567 , 0.08958629, 0.65675706],\n", " [0.3567246 , 0.51081026, 0.13246517],\n", " [0.51665413, 0.25464636, 0.22869956],\n", " [0.22562155, 0.13477427, 0.6396042 ],\n", " [0.41928637, 0.1851027 , 0.3956109 ],\n", " [0.2986142 , 0.1822068 , 0.51917905],\n", " [0.43796587, 0.5088172 , 0.0532169 ],\n", " [0.36305773, 0.5407344 , 0.09620781],\n", " [0.40168276, 0.22597438, 0.3723428 ],\n", " [0.525698 , 0.33632043, 0.13798158],\n", " [0.440705 , 0.37301573, 0.18627922],\n", " [0.3130089 , 0.6254379 , 0.06155315],\n", " [0.23925254, 0.11107812, 0.64966935],\n", " [0.29323807, 0.6405481 , 0.06621383],\n", " [0.6538024 , 0.19487366, 0.15132397],\n", " [0.4068434 , 0.52410513, 0.06905149],\n", " [0.3276567 , 0.6071852 , 0.0651581 ],\n", " [0.3592872 , 0.5445298 , 0.09618303],\n", " [0.5667688 , 0.17972583, 0.2535054 ],\n", " [0.31340864, 0.6314394 , 0.05515193],\n", " [0.55374557, 0.29457554, 0.15167893],\n", " [0.42734364, 0.49750647, 0.07514992],\n", " [0.30575323, 0.63346046, 0.06078625],\n", " [0.51537323, 0.19949993, 0.28512686],\n", " [0.29523924, 0.61226594, 0.09249483],\n", " [0.320049 , 0.6248381 , 0.05511288],\n", " [0.38063663, 0.38691008, 0.2324533 ],\n", " [0.2985715 , 0.64104617, 0.06038239],\n", " [0.30719495, 0.6281224 , 0.06468262],\n", " [0.52452815, 0.38648534, 0.08898648],\n", " [0.6417465 , 0.11507756, 0.24317585],\n", " [0.37231016, 0.43303663, 0.19465321],\n", " [0.30785155, 0.60837 , 0.08377849],\n", " [0.3576193 , 0.39254773, 0.24983294]], dtype=float32)" ] }, "metadata": { "tags": [] }, "execution_count": 52 } ] }, { "cell_type": "code", "metadata": { "id": "abak6XDwKded", "colab_type": "code", "colab": { "base_uri": "https://localhost:8080/", "height": 394 }, "outputId": "ef8b4263-dd93-49f5-dbc0-520a55bff63b" }, "source": [ "HOME_DIR = \"drive/My Drive/\"\n", "frames_classifier = FramesClassifier(\n", " frames_folder = HOME_DIR + \"cs231n-project/datasets/emotiw/test-tiny-local.zip\", \n", " is_test = True,\n", " model_location = HOME_DIR + \"cs231n-project/models/frame-classifier-resnet-lstm-x3.h5\"\n", ")\n", "frames_classifier.predict(\"concatenate_1\")" ], "execution_count": 45, "outputs": [ { "output_type": "stream", "text": [ "FramesClassifier created with is_zip = True, frames_folder = drive/My Drive/cs231n-project/datasets/emotiw/test-tiny-local.zip , is_test = True , model_location = drive/My Drive/cs231n-project/models/frame-classifier-resnet-lstm-x3.h5\n", "Removing existing dir...\n", "Unzipping files to temp dir frames_tmp...\n", "Finished unzipping files\n", "Found 50 frames belonging to 50 videos (test-mode).\n", "Min frames determined to be 13\n", "WARNING:tensorflow:Error in loading the saved optimizer state. As a result, your model is starting with a freshly initialized optimizer.\n", "Customizing model by returning layer concatenate_1\n" ], "name": "stdout" }, { "output_type": "execute_result", "data": { "text/plain": [ "array([[-1.8647227e-03, -3.4038827e-02, 4.3547651e-01, ...,\n", " 1.7179893e-03, 8.8167608e-01, -3.2296131e-05],\n", " [-0.0000000e+00, -8.2707131e-01, 8.4936090e-02, ...,\n", " 1.3595320e-03, 9.7490811e-01, -0.0000000e+00],\n", " [ 3.3392468e-03, -3.6282107e-01, 5.5189729e-01, ...,\n", " 5.7444915e-02, 7.9138809e-01, 0.0000000e+00],\n", " ...,\n", " [-0.0000000e+00, -8.0778056e-01, 3.8071179e-01, ...,\n", " 5.0528154e-02, 9.6614629e-01, 0.0000000e+00],\n", " [-7.2590366e-04, -8.2647181e-01, 2.7033949e-01, ...,\n", " 5.3205504e-03, 9.3420750e-01, 0.0000000e+00],\n", " [-6.7448891e-03, -6.7763430e-01, 4.9857229e-01, ...,\n", " 5.2003324e-02, 9.1682047e-01, 0.0000000e+00]], dtype=float32)" ] }, "metadata": { "tags": [] }, "execution_count": 45 } ] }, { "cell_type": "code", "metadata": { "id": "K-IJzfkSBwoo", "colab_type": "code", "colab": { "base_uri": "https://localhost:8080/", "height": 122 }, "outputId": "f7ca4065-a0c2-4dae-f4bf-1d4d549c5358" }, "source": [ "from google.colab import drive\n", "drive.mount('/content/drive')\n" ], "execution_count": 2, "outputs": [ { "output_type": "stream", "text": [ "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n", "\n", "Enter your authorization code:\n", "··········\n", "Mounted at /content/drive\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "yG4bU_kyCJdv", "colab_type": "code", "colab": {} }, "source": [ "" ], "execution_count": 0, "outputs": [] } ] }