Merge pull request #27 from chrisshyi/master

Merging bug fix for missing audio features in Spotify database

.gitignore

@@ -6,3 +6,6 @@ db.sqlite3
 
 api-keys.sh
 Pipfile
+super-pass.txt
+*.js
+*.ini

musicvis/settings.py

@@ -76,8 +76,12 @@ WSGI_APPLICATION = 'musicvis.wsgi.application'
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'NAME': 'spotifyvis',
+        'USER': 'django',
+        'PASSWORD': 'django',
+        'HOST': 'localhost',
+        'PORT': '', 
     }
 }
 

requirements.txt

@@ -7,6 +7,7 @@ idna==2.6
 isort==4.3.4
 lazy-object-proxy==1.3.1
 mccabe==0.6.1
+psycopg2==2.7.4
 pylint==1.8.4
 pytz==2018.4
 requests==2.18.4

spotifyvis/migrations/0001_initial.py

@@ -0,0 +1,85 @@
+# Generated by Django 2.0.5 on 2018-06-03 23:01
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Artist',
+            fields=[
+                ('artist_id', models.CharField(max_length=30, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=50, unique=True)),
+                ('genre', models.CharField(max_length=20)),
+            ],
+            options={
+                'verbose_name': 'Artist',
+                'verbose_name_plural': 'Artists',
+            },
+        ),
+        migrations.CreateModel(
+            name='Track',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('track_id', models.CharField(max_length=30)),
+                ('year', models.PositiveSmallIntegerField()),
+                ('popularity', models.DecimalField(decimal_places=2, max_digits=2)),
+                ('runtime', models.PositiveSmallIntegerField()),
+                ('name', models.CharField(max_length=75)),
+            ],
+            options={
+                'verbose_name': 'Track',
+                'verbose_name_plural': 'Tracks',
+            },
+        ),
+        migrations.CreateModel(
+            name='User',
+            fields=[
+                ('user_id', models.CharField(max_length=30, primary_key=True, serialize=False)),
+                ('username', models.CharField(max_length=30)),
+            ],
+            options={
+                'verbose_name': 'User',
+                'verbose_name_plural': 'Users',
+            },
+        ),
+        migrations.CreateModel(
+            name='AudioFeatures',
+            fields=[
+                ('track', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='spotifyvis.Track')),
+                ('danceability', models.DecimalField(decimal_places=2, max_digits=2)),
+                ('energy', models.DecimalField(decimal_places=2, max_digits=2)),
+                ('loudness', models.DecimalField(decimal_places=2, max_digits=2)),
+                ('speechiness', models.DecimalField(decimal_places=2, max_digits=2)),
+                ('acousticness', models.DecimalField(decimal_places=2, max_digits=2)),
+                ('instrumentalness', models.DecimalField(decimal_places=2, max_digits=2)),
+                ('valence', models.DecimalField(decimal_places=2, max_digits=2)),
+                ('tempo', models.DecimalField(decimal_places=2, max_digits=2)),
+            ],
+            options={
+                'verbose_name': 'AudioFeatures',
+                'verbose_name_plural': 'AudioFeatures',
+            },
+        ),
+        migrations.AddField(
+            model_name='track',
+            name='artist',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='spotifyvis.Artist'),
+        ),
+        migrations.AddField(
+            model_name='track',
+            name='users',
+            field=models.ManyToManyField(to='spotifyvis.User'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='track',
+            unique_together={('track_id', 'artist')},
+        ),
+    ]

spotifyvis/models.py

@@ -1,3 +1,66 @@
 from django.db import models
 
-# Create your models here.
+
+class Artist(models.Model):
+    class Meta:
+        verbose_name = "Artist"
+        verbose_name_plural = "Artists"
+
+    artist_id = models.CharField(primary_key=True, max_length=30)
+    # unique since only storing one genre per artist right now
+    name = models.CharField(unique=True, max_length=50)
+    genre = models.CharField(max_length=20)
+
+    def __str__(self):
+        return self.name
+
+
+class User(models.Model):
+    class Meta:
+        verbose_name = "User"
+        verbose_name_plural = "Users"
+
+    user_id = models.CharField(primary_key=True, max_length=30) # the user's Spotify ID
+    username = models.CharField(max_length=30) # User's Spotify user name, if set
+
+    def __str__(self):
+        return self.username
+
+
+class Track(models.Model):
+    
+    class Meta:
+        verbose_name = "Track"
+        verbose_name_plural = "Tracks"
+        unique_together = ('track_id', 'artist',)
+
+    track_id = models.CharField(max_length=30)
+    artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
+    year = models.PositiveSmallIntegerField()
+    popularity = models.DecimalField(decimal_places=2, max_digits=2)
+    runtime = models.PositiveSmallIntegerField()
+    name = models.CharField(max_length=75)
+    users = models.ManyToManyField(User)
+
+    def __str__(self):
+        return self.name 
+
+
+class AudioFeatures(models.Model):
+        
+    class Meta:
+        verbose_name = "AudioFeatures"
+        verbose_name_plural = "AudioFeatures"
+
+    track = models.OneToOneField(Track, on_delete=models.CASCADE, primary_key=True,)
+    danceability = models.DecimalField(decimal_places=2, max_digits=2)
+    energy = models.DecimalField(decimal_places=2, max_digits=2)
+    loudness = models.DecimalField(decimal_places=2, max_digits=2)
+    speechiness = models.DecimalField(decimal_places=2, max_digits=2)
+    acousticness = models.DecimalField(decimal_places=2, max_digits=2)
+    instrumentalness = models.DecimalField(decimal_places=2, max_digits=2)
+    valence = models.DecimalField(decimal_places=2, max_digits=2)
+    tempo = models.DecimalField(decimal_places=2, max_digits=2)
+
+    def __str__(self):
+        return super(AudioFeatures, self).__str__()

spotifyvis/static/spotifyvis/scripts/index.js

@@ -0,0 +1,42 @@
+document.getElementById("login-btn").addEventListener("click", function() {
+    let httpRequest = new XMLHttpRequest();
+
+    /*
+     * Handler for the response
+     */
+    httpRequest.onreadystatechange = function() {
+        if (httpRequest.readyState === XMLHttpRequest.DONE) {
+            if (httpRequest.status === 200) {
+                // hide the login button
+                document.getElementById('login').setAttribute("display", "none");
+
+                let responseData = JSON.parse(httpRequest.responseText);
+                let dataList = document.getElementById("data-list");
+                
+
+                for (let key in responseData) {
+                    let newLi = document.createElement("li");
+                    let innerList = document.createElement("ul");
+                    
+                    let dataLabel = document.createElement("li");
+                    dataLabel.innerText = key;
+
+                    let dataValue = document.createElement("li");
+                    dataValue.innerText = responseData[key];
+
+                    innerList.appendChild(dataLabel);
+                    innerList.appendChild(dataValue);
+
+                    newLi.appendChild(innerList);
+                    dataList.appendChild(newLi);
+                }
+            } else {
+                alert("There was a problem with the login request, please try again!");
+            }
+        }
+    }
+
+    httpRequest.open('GET', '/login', true);
+    httpRequest.send();
+});
+

spotifyvis/templates/spotifyvis/index.html

@@ -1,4 +1,5 @@
 <!DOCTYPE html>
+{% load static %}
 <html>
   <head>
     <title>User Login</title>
@@ -11,14 +12,25 @@
         width: 500px;
       }
     </style>
+
   </head>
 
   <body>
     <div class="container">
       <div id="login">
         <h1>This is an example of the Authorization Code flow</h1>
-        <a href="/login" class="btn btn-primary">Log in with Spotify</a>
+        <a href="/login" class="btn btn-primary">Log In (Original)</a>
+        <button id="login-btn">Log In</button>
+      </div>
+
+      <div id="data-container">
+        <ul id="data-list">
+
+        </ul>
       </div>
     </div>
+
+
+    <script src="{% static 'spotifyvis/scripts/index.js' %}"></script>
   </body>
 </html>

spotifyvis/templates/spotifyvis/user_data.html

@@ -14,7 +14,13 @@
         <!--[if lt IE 7]>
             <p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="#">upgrade your browser</a> to improve your experience.</p>
         <![endif]-->
-        <h1>Logged in as {{ id }}</h1>
+        <p>Logged in as {{ id }}</p>
         <h2>Display name {{ user_name }}</h2>
+				<ul>
+					{% for genre_name, genre_count in genre_dict.items %}
+						<li>{{ genre_name }} - {{ genre_count }}</li>
+					{% endfor %}
+                </ul>
+        <script src="{% static 'spotifyvis/scripts/user_data.js' %}"></script>
     </body>
 </html>

spotifyvis/utils.py

@@ -1,15 +1,17 @@
 import requests
 import math
 import pprint
+from .models import Artist, User, Track, AudioFeatures
 
 #  parse_library {{{ # 
 
-def parse_library(headers, tracks, library_stats):
+def parse_library(headers, tracks, library_stats, user):
     """Scans user's library for certain number of tracks to update library_stats with.
 
     :headers: For API call.
     :tracks: Number of tracks to get from user's library.
     :library_stats: Dictionary containing the data mined from user's library 
+    :user: a User object representing the user whose library we are parsing
 
     :returns: None
 
@@ -20,19 +22,25 @@ def parse_library(headers, tracks, library_stats):
     # keeps track of point to get songs from
     offset = 0
     payload = {'limit': str(limit)}
+    # use two separate variables to track, because the average popularity also requires num_samples 
+    num_samples = 0  # number of actual track samples
+    feature_data_points = 0  # number of feature data analyses (some tracks do not have analyses available)
+
     for _ in range(0, tracks, limit):
         payload['offset'] = str(offset)
         saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', headers=headers, params=payload).json()
-        num_samples = offset
         for track_dict in saved_tracks_response['items']:
-            # Track the number of samples for calculating
-            # audio feature averages and standard deviations on the fly
             num_samples += 1 
             get_track_info(track_dict['track'], library_stats, num_samples)
             #  get_genre(headers, track_dict['track']['album']['id'])
             audio_features_dict = get_audio_features(headers, track_dict['track']['id'])
-            for feature, feature_data in audio_features_dict.items():
-                update_audio_feature_stats(feature, feature_data, num_samples, library_stats)
+            if len(audio_features_dict) != 0:
+                # Track the number of audio analyses for calculating
+                # audio feature averages and standard deviations on the fly
+                feature_data_points += 1
+                
+                for feature, feature_data in audio_features_dict.items():
+                    update_audio_feature_stats(feature, feature_data, feature_data_points, library_stats)
             for artist_dict in track_dict['track']['artists']:
                 increase_artist_count(headers, artist_dict['name'], artist_dict['id'], library_stats)
         # calculates num_songs with offset + songs retrieved
@@ -51,10 +59,13 @@ def get_audio_features(headers, track_id):
         track_id: the id of the soundtrack, needed to query the Spotify API
         
     Returns:
-        A dictionary with the features as its keys
+        A dictionary with the features as its keys, if audio feature data is missing for the track, 
+        an empty dictionary is returned.
     """
     
     response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json()
+    if 'error' in response:
+        return {}
     features_dict = {}
 
     # Data that we don't need

spotifyvis/views.py

@@ -9,9 +9,10 @@ import json
 import pprint
 from datetime import datetime
 from .utils import parse_library, process_library_stats
+from .models import User, Track, AudioFeatures, Artist 
 
 TIME_FORMAT = '%Y-%m-%d-%H-%M-%S'
-library_stats = {"audio_features":{}, "genres":{}, "year_released":{}, "artists":{}, "num_songs":0, "popularity":[], "total_runtime":0}
+TRACKS_TO_QUERY = 5
 
 #  generate_random_string {{{ # 
 
@@ -133,12 +134,18 @@ def user_data(request):
     }
 
     user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json()
-    context = {
-        'user_name': user_data_response['display_name'],
-        'id': user_data_response['id'],
-    }
+    request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model
+    request.session['user_name'] = user_data_response['display_name']
+    user = None # will be set to the current user object later
+    try:
+        user = User.objects.get(user_id=request.session['user_id'])
+    except User.DoesNotExist:
+        user = User.objects.create(user_id=request.session['user_id'], user_name=request.session['user_name'])
+    # context = {
+    #     'user_name': user_data_response['display_name'],
+    #     'id': user_data_response['id'],
+    # }
 
-    tracks_to_query = 5
     library_stats = {
         "audio_features":{}, 
         "genres":{}, 
@@ -151,7 +158,7 @@ def user_data(request):
         },   
         "total_runtime": 0
     }
-    parse_library(headers, tracks_to_query, library_stats)
+    parse_library(headers, TRACKS_TO_QUERY, library_stats, user)
     processed_library_stats = process_library_stats(library_stats)
     print("================================================")
     print("Processed data follows\n")