Merge pull request #30 from Kevin-Mok/in-progress

- pass data from database to D3 JS
- deleted migration history
- modified `views.py, utils.py` to be more directed toward using database vs. `library_stats`

spotifyvis/migrations/0001_initial.py

@@ -1,79 +0,0 @@
-# Generated by Django 2.0.5 on 2018-06-06 07:26
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    dependencies = [
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='Artist',
-            fields=[
-                ('artist_id', models.CharField(max_length=30, primary_key=True, serialize=False)),
-                ('name', models.CharField(max_length=50, unique=True)),
-                ('genre', models.CharField(max_length=20)),
-            ],
-            options={
-                'verbose_name_plural': 'Artists',
-                'verbose_name': 'Artist',
-            },
-        ),
-        migrations.CreateModel(
-            name='Track',
-            fields=[
-                ('track_id', models.CharField(max_length=30, primary_key=True, serialize=False)),
-                ('year', models.PositiveSmallIntegerField()),
-                ('popularity', models.PositiveSmallIntegerField()),
-                ('runtime', models.PositiveSmallIntegerField()),
-                ('name', models.CharField(max_length=75)),
-            ],
-            options={
-                'verbose_name_plural': 'Tracks',
-                'verbose_name': 'Track',
-            },
-        ),
-        migrations.CreateModel(
-            name='User',
-            fields=[
-                ('user_id', models.CharField(max_length=30, primary_key=True, serialize=False)),
-            ],
-            options={
-                'verbose_name_plural': 'Users',
-                'verbose_name': 'User',
-            },
-        ),
-        migrations.CreateModel(
-            name='AudioFeatures',
-            fields=[
-                ('track', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='spotifyvis.Track')),
-                ('danceability', models.DecimalField(decimal_places=2, max_digits=2)),
-                ('energy', models.DecimalField(decimal_places=2, max_digits=2)),
-                ('loudness', models.DecimalField(decimal_places=2, max_digits=2)),
-                ('speechiness', models.DecimalField(decimal_places=2, max_digits=2)),
-                ('acousticness', models.DecimalField(decimal_places=2, max_digits=2)),
-                ('instrumentalness', models.DecimalField(decimal_places=2, max_digits=2)),
-                ('valence', models.DecimalField(decimal_places=2, max_digits=2)),
-                ('tempo', models.DecimalField(decimal_places=2, max_digits=2)),
-            ],
-            options={
-                'verbose_name_plural': 'AudioFeatures',
-                'verbose_name': 'AudioFeatures',
-            },
-        ),
-        migrations.AddField(
-            model_name='track',
-            name='artists',
-            field=models.ManyToManyField(blank=True, to='spotifyvis.Artist'),
-        ),
-        migrations.AddField(
-            model_name='track',
-            name='users',
-            field=models.ManyToManyField(blank=True, to='spotifyvis.User'),
-        ),
-    ]

spotifyvis/migrations/0002_auto_20180606_0523.py

@@ -1,53 +0,0 @@
-# Generated by Django 2.0.5 on 2018-06-06 09:23
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('spotifyvis', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='acousticness',
-            field=models.DecimalField(decimal_places=3, max_digits=3),
-        ),
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='danceability',
-            field=models.DecimalField(decimal_places=3, max_digits=3),
-        ),
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='energy',
-            field=models.DecimalField(decimal_places=3, max_digits=3),
-        ),
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='instrumentalness',
-            field=models.DecimalField(decimal_places=3, max_digits=3),
-        ),
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='loudness',
-            field=models.DecimalField(decimal_places=3, max_digits=3),
-        ),
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='speechiness',
-            field=models.DecimalField(decimal_places=3, max_digits=3),
-        ),
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='tempo',
-            field=models.DecimalField(decimal_places=3, max_digits=3),
-        ),
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='valence',
-            field=models.DecimalField(decimal_places=3, max_digits=3),
-        ),
-    ]

spotifyvis/migrations/0003_auto_20180606_0525.py

@@ -1,23 +0,0 @@
-# Generated by Django 2.0.5 on 2018-06-06 09:25
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('spotifyvis', '0002_auto_20180606_0523'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='loudness',
-            field=models.DecimalField(decimal_places=3, max_digits=6),
-        ),
-        migrations.AlterField(
-            model_name='audiofeatures',
-            name='tempo',
-            field=models.DecimalField(decimal_places=3, max_digits=6),
-        ),
-    ]

spotifyvis/models.py

@@ -1,19 +1,19 @@
 from django.db import models
 
 # id's are 22 in length in examples but set to 30 for buffer
-id_length=30
-
+MAX_ID = 30
 #  Artist {{{ # 
 
+
 class Artist(models.Model):
     class Meta:
         verbose_name = "Artist"
         verbose_name_plural = "Artists"
 
-    artist_id = models.CharField(primary_key=True, max_length=id_length)
+    artist_id = models.CharField(primary_key=True, max_length=MAX_ID)
     # unique since only storing one genre per artist right now
     name = models.CharField(unique=True, max_length=50)
-    genre = models.CharField(max_length=20)
+    genre = models.CharField(max_length=30)
 
     def __str__(self):
         return self.name
@@ -27,7 +27,7 @@ class User(models.Model):
         verbose_name = "User"
         verbose_name_plural = "Users"
 
-    user_id = models.CharField(primary_key=True, max_length=id_length) # the user's Spotify ID
+    user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID
     #  username = models.CharField(max_length=30) # User's Spotify user name, if set
 
     def __str__(self):
@@ -42,15 +42,14 @@ class Track(models.Model):
     class Meta:
         verbose_name = "Track"
         verbose_name_plural = "Tracks"
-        #  unique_together = ('track_id', 'artist',)
 
-    track_id = models.CharField(primary_key=True, max_length=id_length)
+    track_id = models.CharField(primary_key=True, max_length=MAX_ID)
     #  artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
     artists = models.ManyToManyField(Artist, blank=True)
     year = models.PositiveSmallIntegerField()
     popularity = models.PositiveSmallIntegerField()
     runtime = models.PositiveSmallIntegerField()
-    name = models.CharField(max_length=75)
+    name = models.CharField(max_length=150)
     users = models.ManyToManyField(User, blank=True)
 
     def __str__(self):
@@ -60,6 +59,7 @@ class Track(models.Model):
 
 #  AudioFeatures {{{ # 
 
+
 class AudioFeatures(models.Model):
         
     class Meta:
@@ -79,4 +79,4 @@ class AudioFeatures(models.Model):
     def __str__(self):
         return super(AudioFeatures, self).__str__()
 
-#  }}} AudioFeatures # 
+#  }}} AudioFeatures #

spotifyvis/templates/spotifyvis/index.html

@@ -20,6 +20,7 @@
       <div id="login">
         <h1>This is an example of the Authorization Code flow</h1>
         <a href="/login" class="btn btn-primary">Log In (Original)</a>
+		<a href="/test_db" class="btn btn-primary">Test DB</a>
         <button id="login-btn">Log In</button>
       </div>
 

spotifyvis/templates/spotifyvis/test_db.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
+<!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
+<!--[if IE 8]>         <html class="no-js lt-ie9"> <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js"> <!--<![endif]-->
+    <head>
+        <meta charset="utf-8">
+        <meta http-equiv="X-UA-Compatible" content="IE=edge">
+        <title>Test DB Page</title>
+        <meta name="description" content="">
+        <meta name="viewport" content="width=device-width, initial-scale=1">
+    </head>
+    <body>
+        <!--[if lt IE 7]>
+            <p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="#">upgrade your browser</a> to improve your experience.</p>
+        <![endif]-->
+        <ul>
+          {% for artist in artist_data %}
+              <li>{{ artist.name }} - {{ artist.num_songs }}</li>
+          {% endfor %}
+        </ul>
+        <pre> {% filter force_escape %} {% debug %} {% endfilter %} </pre>
+      <script src="https://d3js.org/d3.v5.min.js"></script>
+      <script>
+        d3.json("{% url "get_artist_data" user_id %}").then(function(data) {
+            data.forEach(function(d) {
+                console.log(d.name, d.num_songs);
+            });
+        });
+      </script>
+    </body>
+</html>

spotifyvis/templates/spotifyvis/user_data.html

@@ -1,3 +1,4 @@
+{% load static %}
 <!DOCTYPE html>
 <!--[if lt IE 7]>      <html class="no-js lt-ie9 lt-ie8 lt-ie7"> <![endif]-->
 <!--[if IE 7]>         <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
@@ -20,6 +21,6 @@
 					{% for genre_name, genre_count in genre_dict.items %}
 						<li>{{ genre_name }} - {{ genre_count }}</li>
 					{% endfor %}
-                </ul>
+        </ul>
     </body>
 </html>

spotifyvis/urls.py

@@ -1,9 +1,13 @@
 from django.urls import path, include
-from . import views
+from django.conf.urls import url
+
+from .views import *
 
 urlpatterns = [
-    path('', views.index, name='index'),
-    path('login', views.login, name='login'),
-    path('callback', views.callback, name='callback'),
-    path('user_data', views.user_data, name='user_data'),
-]
+    path('', index, name='index'),
+    path('login', login, name='login'),
+    path('callback', callback, name='callback'),
+    path('user_data', user_data, name='user_data'),
+    path('test_db', test_db, name='test_db'),
+    path('user_artists/<str:user_id>', get_artist_data, name='get_artist_data'),
+]

spotifyvis/utils.py

@@ -1,9 +1,13 @@
 #  imports {{{ # 
-
 import requests
 import math
 import pprint
+
 from .models import Artist, User, Track, AudioFeatures
+from django.db.models import Count
+from django.http import JsonResponse
+from django.core import serializers
+import json
 
 #  }}} imports # 
 
@@ -28,7 +32,6 @@ def parse_library(headers, tracks, library_stats, user):
     payload = {'limit': str(limit)}
     # use two separate variables to track, because the average popularity also requires num_samples 
     num_samples = 0  # number of actual track samples
-    feature_data_points = 0  # number of feature data analyses (some tracks do not have analyses available)
 
     # iterate until hit requested num of tracks
     for _ in range(0, tracks, limit):
@@ -39,22 +42,27 @@ def parse_library(headers, tracks, library_stats, user):
         # TODO: refactor the for loop body into helper function
         # iterate through each track
         for track_dict in saved_tracks_response['items']:
-            num_samples += 1 
             # update artist info before track so that Track object can reference
             # Artist object
             track_artists = []
             for artist_dict in track_dict['track']['artists']:
-                increase_artist_count(headers, artist_dict['name'], 
-                        artist_dict['id'], library_stats)
-                track_artists.append(Artist.objects.get_or_create(
+                artist_obj, artist_created = Artist.objects.get_or_create(
                     artist_id=artist_dict['id'],
                     name=artist_dict['name'],
-                    )[0])
+                    )
+
+                update_artist_genre(headers, artist_obj) 
+                # get_or_create() returns a tuple (obj, created)
+                track_artists.append(artist_obj)
             
-            track_obj = save_track_obj(track_dict['track'], track_artists, user)
-            get_track_info(track_dict['track'], library_stats, num_samples)
-            audio_features_dict = get_audio_features(headers,
-                    track_dict['track']['id'], track_obj)
+            track_obj, track_created = save_track_obj(track_dict['track'], track_artists, user)
+
+            # if a new track is not created, the associated audio feature does not need to be created again
+            if track_created:
+                save_audio_features(headers, track_dict['track']['id'], track_obj)
+            """
+            TODO: Put this logic in another function
+            # Audio analysis could be empty if not present in Spotify database
             if len(audio_features_dict) != 0:
                 # Track the number of audio analyses for calculating
                 # audio feature averages and standard deviations on the fly
@@ -62,62 +70,56 @@ def parse_library(headers, tracks, library_stats, user):
                 for feature, feature_data in audio_features_dict.items():
                     update_audio_feature_stats(feature, feature_data, 
                             feature_data_points, library_stats)
-
+            """
         # calculates num_songs with offset + songs retrieved
-        library_stats['num_songs'] = offset + len(saved_tracks_response['items'])
         offset += limit
-    calculate_genres_from_artists(headers, library_stats)
+    # calculate_genres_from_artists(headers, library_stats)
     #  pprint.pprint(library_stats)
 
 #  }}} parse_library # 
 
 #  save_track_obj {{{ # 
 
+
 def save_track_obj(track_dict, artists, user):
     """Make an entry in the database for this track if it doesn't exist already.
 
     :track_dict: dictionary from the API call containing track information.
     :artists: artists of the song, passed in as a list of Artist objects.
     :user: User object for which this Track is to be associated with.
-    :returns: The created/retrieved Track object.
+    :returns: (The created/retrieved Track object, created) 
 
     """
-    track_obj_query = Track.objects.filter(track_id__exact=track_dict['id'])
-    if len(track_obj_query) == 0:
-        new_track = Track.objects.create(
-            track_id=track_dict['id'],
-            year=track_dict['album']['release_date'].split('-')[0],
-            popularity=int(track_dict['popularity']),
-            runtime=int(float(track_dict['duration_ms']) / 1000),
-            name=track_dict['name'],
-        )
-        #  print("pop/run: ", new_track.popularity, new_track.runtime)
-
-        # have to add artists and user after saving object since track needs to
-        # have ID before filling in m2m field
+    print(track_dict['name'])
+    new_track, created = Track.objects.get_or_create(
+        track_id=track_dict['id'],
+        year=track_dict['album']['release_date'].split('-')[0],
+        popularity=int(track_dict['popularity']),
+        runtime=int(float(track_dict['duration_ms']) / 1000),
+        name=track_dict['name'],
+    )
+
+    # have to add artists and user after saving object since track needs to
+    # have ID before filling in m2m field
+    if created:
         for artist in artists:
             new_track.artists.add(artist)
         new_track.users.add(user)
         new_track.save()
-        return new_track
-    elif len(track_obj_query) == 1:
-        return track_obj_query[0]
+    return new_track, created
 
 #  }}} save_track_obj # 
 
 #  get_audio_features {{{ # 
 
-def get_audio_features(headers, track_id, track):
-    """Returns the audio features of a soundtrack
+def save_audio_features(headers, track_id, track):
+    """Creates and saves a new AudioFeatures object
 
     Args:
         headers: headers containing the API token
         track_id: the id of the soundtrack, needed to query the Spotify API
-        track: Track object to associate with the AudioFeatures object
+        track: Track object to associate with the new AudioFeatures object
         
-    Returns:
-        A dictionary with the features as its keys, if audio feature data is missing for the track, 
-        an empty dictionary is returned.
     """
     
     response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json()
@@ -137,7 +139,6 @@ def get_audio_features(headers, track_id, track):
             setattr(audio_features_entry, key, val)
     audio_features_entry.save()
 
-    return features_dict
 
 #  }}} get_audio_features # 
 
@@ -296,30 +297,28 @@ def get_track_info(track_dict, library_stats, sample_size):
 
 #  }}} get_track_info # 
 
-#  calculate_genres_from_artists {{{ # 
+#  update_genres_from_artists {{{ # 
+
 
-def calculate_genres_from_artists(headers, library_stats):
-    """Tallies up genre counts based on artists in library_stats.
+def update_artist_genre(headers, artist_obj):
+    """Updates the top genre for an artist by querying the Spotify API
 
     :headers: For making the API call.
-    :library_stats: Dictionary containing the data mined from user's Spotify library
+    :artist_obj: the Artist object whose genre field will be updated 
 
     :returns: None
 
     """
-    for artist_entry in library_stats['artists'].values():
-        artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_entry['id'], headers=headers).json()
-        # increase each genre count by artist count
-        for genre in artist_response['genres']:
-            increase_nested_key('genres', genre, library_stats, artist_entry['count'])
-
-        # update genre for artist in database with top genre
-        Artist.objects.filter(artist_id=artist_entry['id']).update(genre=artist_response['genres'][0])
+    artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_obj.artist_id, headers=headers).json()
+    # update genre for artist in database with top genre
+    artist_obj.genre = artist_response['genres'][0]
+    artist_obj.save()
 
 #  }}} calculate_genres_from_artists # 
 
 #  process_library_stats {{{ # 
 
+
 def process_library_stats(library_stats):
     """Processes library_stats into format more suitable for D3 consumption
 
@@ -372,3 +371,15 @@ def process_library_stats(library_stats):
     return processed_library_stats
 
 #  }}} process_library_stats # 
+
+def get_genre_data(user):
+    """Return genre data needed to create the graph user.
+
+    :user: User object for which to return the data for.
+    :returns: List of dicts containing counts for each genre.
+
+    """
+    pass
+    #  user_tracks = Track.objects.filter(users__exact=user)
+    #  for track in user_tracks:
+        #  print(track.name)

spotifyvis/views.py

@@ -1,7 +1,5 @@
 #  imports {{{ # 
 
-from django.shortcuts import render, redirect
-from django.http import HttpResponse, HttpResponseBadRequest
 import math
 import random
 import requests
@@ -10,7 +8,11 @@ import urllib
 import json
 import pprint
 from datetime import datetime
-from .utils import parse_library, process_library_stats
+
+from django.shortcuts import render, redirect
+from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse
+from django.db.models import Count
+from .utils import parse_library, process_library_stats 
 from .models import User, Track, AudioFeatures, Artist 
 
 #  }}} imports # 
@@ -20,6 +22,7 @@ TRACKS_TO_QUERY = 5
 
 #  generate_random_string {{{ # 
 
+
 def generate_random_string(length):
     """Generates a random string of a certain length
 
@@ -41,6 +44,7 @@ def generate_random_string(length):
 
 #  token_expired {{{ # 
 
+
 def token_expired(token_obtained_at, valid_for):
     """Returns True if token expired, False if otherwise
 
@@ -140,6 +144,8 @@ def user_data(request):
     user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json()
     request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model
     #  request.session['user_name'] = user_data_response['display_name']
+
+    # get_or_create() returns a tuple (obj, created)
     user = User.objects.get_or_create(user_id=user_data_response['id'])[0]
 
     context = {
@@ -167,3 +173,24 @@ def user_data(request):
     return render(request, 'spotifyvis/user_data.html', context)
 
 #  }}} user_data  # 
+
+def test_db(request):
+    user_id = "polarbier"
+    context = {
+        'user_id': user_id,
+    }
+    #  get_artist_data(user)
+    return render(request, 'spotifyvis/test_db.html', context)
+
+
+def get_artist_data(request, user_id):
+
+    # TODO: not actual artists for user
+    # PICK UP: figure out how to pass data to D3/frontend
+    print(user_id)
+    #  user = User.objects.get(user_id=user_id)
+    artist_counts = Artist.objects.annotate(num_songs=Count('track'))
+    processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts]
+    #  for artist in artist_counts:
+        #  print(artist.name, artist.num_songs)
+    return JsonResponse(data=processed_artist_data, safe=False)