From 797115e9cb07b7aa9dbfe48f2e750e93fd754ab0 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 8 Jun 2018 18:45:16 -0400 Subject: [PATCH] Clean up utils.py to use database Uses of the original library_stats dictionary in utils.py were either removed or commented out, replaced by the saving of models into the database. --- spotifyvis/models.py | 45 ++++++- .../static/spotifyvis/scripts/test_db.js | 8 +- spotifyvis/templates/spotifyvis/test_db.html | 11 +- spotifyvis/utils.py | 113 +++++++----------- spotifyvis/views.py | 21 +++- 5 files changed, 110 insertions(+), 88 deletions(-) diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 19a99be..aacfd1e 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -1,8 +1,7 @@ from django.db import models # id's are 22 in length in examples but set to 30 for buffer -id_length=30 - +MAX_ID = 30 # Artist {{{ # class Artist(models.Model): @@ -10,7 +9,7 @@ class Artist(models.Model): verbose_name = "Artist" verbose_name_plural = "Artists" - artist_id = models.CharField(primary_key=True, max_length=id_length) + artist_id = models.CharField(primary_key=True, max_length=MAX_ID) # unique since only storing one genre per artist right now name = models.CharField(unique=True, max_length=50) genre = models.CharField(max_length=20) @@ -27,7 +26,7 @@ class User(models.Model): verbose_name = "User" verbose_name_plural = "Users" - user_id = models.CharField(primary_key=True, max_length=id_length) # the user's Spotify ID + user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID # username = models.CharField(max_length=30) # User's Spotify user name, if set def __str__(self): @@ -43,7 +42,7 @@ class Track(models.Model): verbose_name = "Track" verbose_name_plural = "Tracks" - track_id = models.CharField(primary_key=True, max_length=id_length) + track_id = models.CharField(primary_key=True, max_length=MAX_ID) # artist = models.ForeignKey(Artist, on_delete=models.CASCADE) artists = models.ManyToManyField(Artist, blank=True) year = models.PositiveSmallIntegerField() @@ -79,3 +78,39 @@ class AudioFeatures(models.Model): return super(AudioFeatures, self).__str__() # }}} AudioFeatures # + + +''' +class UserAudioSummary(models.Model): + """ + Stores the summary of a user's audio preferences + """ + class Meta: + verbose_name = "AudioFeatures" + verbose_name_plural = "AudioFeatures" + + user = models.OneToOneField(User, on_delete=models.CASCADE, primary_key=True,) + avg_acousticness = models.DecimalField(decimal_places=3, max_digits=3) + stdev_acousticness = models.DecimalField(decimal_places=3, max_digits=3) + + avg_danceability = models.DecimalField(decimal_places=3, max_digits=3) + stdev_danceability = models.DecimalField(decimal_places=3, max_digits=3) + + avg_energy = models.DecimalField(decimal_places=3, max_digits=3) + stdev_energy = models.DecimalField(decimal_places=3, max_digits=3) + + avg_instrumentalness = models.DecimalField(decimal_places=3, max_digits=3) + stdev_instrumentalness = models.DecimalField(decimal_places=3, max_digits=3) + + avg_loudness = models.DecimalField(decimal_places=3, max_digits=6) + stdev_loudness = models.DecimalField(decimal_places=3, max_digits=6) + + avg_speechiness = models.DecimalField(decimal_places=3, max_digits=3) + stdev_speechiness = models.DecimalField(decimal_places=3, max_digits=3) + + avg_tempo = models.DecimalField(decimal_places=3, max_digits=6) + stdev_tempo = models.DecimalField(decimal_places=3, max_digits=6) + + avg_valence = models.DecimalField(decimal_places=3, max_digits=3) + stdev_valence = models.DecimalField(decimal_places=3, max_digits=3) +''' \ No newline at end of file diff --git a/spotifyvis/static/spotifyvis/scripts/test_db.js b/spotifyvis/static/spotifyvis/scripts/test_db.js index 6830820..cb5fb67 100644 --- a/spotifyvis/static/spotifyvis/scripts/test_db.js +++ b/spotifyvis/static/spotifyvis/scripts/test_db.js @@ -1,10 +1,4 @@ -console.log("{{ user_id }}"); -artist_data = JSON.parse('{{ artist_data }}'); -artist_data.forEach(function(d) { - console.log(d.name, d.num_songs); -}); - -d3.json("{% url "get_artist_data" user_id %}", function(error, data) { +d3.json("{% url "get_artist_data", user_id %}").then(function(error, data) { data.forEach(function(d) { console.log(d.name, d.num_songs); }); diff --git a/spotifyvis/templates/spotifyvis/test_db.html b/spotifyvis/templates/spotifyvis/test_db.html index 3f7a0d0..0c51c81 100644 --- a/spotifyvis/templates/spotifyvis/test_db.html +++ b/spotifyvis/templates/spotifyvis/test_db.html @@ -20,8 +20,13 @@ {% endfor %} 
 {% filter force_escape %} {% debug %} {% endfilter %}
- - {% load static %} - + + diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index dc684e3..1d61a9e 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -1,5 +1,4 @@ # imports {{{ # - import requests import math import pprint @@ -8,6 +7,7 @@ from .models import Artist, User, Track, AudioFeatures from django.db.models import Count from django.http import JsonResponse from django.core import serializers +import json # }}} imports # @@ -32,7 +32,6 @@ def parse_library(headers, tracks, library_stats, user): payload = {'limit': str(limit)} # use two separate variables to track, because the average popularity also requires num_samples num_samples = 0 # number of actual track samples - feature_data_points = 0 # number of feature data analyses (some tracks do not have analyses available) # iterate until hit requested num of tracks for _ in range(0, tracks, limit): @@ -43,22 +42,27 @@ def parse_library(headers, tracks, library_stats, user): # TODO: refactor the for loop body into helper function # iterate through each track for track_dict in saved_tracks_response['items']: - num_samples += 1 # update artist info before track so that Track object can reference # Artist object track_artists = [] for artist_dict in track_dict['track']['artists']: - increase_artist_count(headers, artist_dict['name'], - artist_dict['id'], library_stats) - track_artists.append(Artist.objects.get_or_create( + artist_obj, artist_created = Artist.objects.get_or_create( artist_id=artist_dict['id'], name=artist_dict['name'], - )[0]) + ) + + update_artist_genre(headers, artist_obj) + # get_or_create() returns a tuple (obj, created) + track_artists.append(artist_obj) - track_obj = save_track_obj(track_dict['track'], track_artists, user) - get_track_info(track_dict['track'], library_stats, num_samples) - audio_features_dict = get_audio_features(headers, - track_dict['track']['id'], track_obj) + track_obj, track_created = save_track_obj(track_dict['track'], track_artists, user) + + # if a new track is not created, the associated audio feature does not need to be created again + if track_created: + save_audio_features(headers, track_dict['track']['id'], track_obj) + """ + TODO: Put this login in another function + # Audio analysis could be empty if not present in Spotify database if len(audio_features_dict) != 0: # Track the number of audio analyses for calculating # audio feature averages and standard deviations on the fly @@ -66,62 +70,55 @@ def parse_library(headers, tracks, library_stats, user): for feature, feature_data in audio_features_dict.items(): update_audio_feature_stats(feature, feature_data, feature_data_points, library_stats) - + """ # calculates num_songs with offset + songs retrieved - library_stats['num_songs'] = offset + len(saved_tracks_response['items']) offset += limit - calculate_genres_from_artists(headers, library_stats) + # calculate_genres_from_artists(headers, library_stats) # pprint.pprint(library_stats) # }}} parse_library # # save_track_obj {{{ # + def save_track_obj(track_dict, artists, user): """Make an entry in the database for this track if it doesn't exist already. :track_dict: dictionary from the API call containing track information. :artists: artists of the song, passed in as a list of Artist objects. :user: User object for which this Track is to be associated with. - :returns: The created/retrieved Track object. + :returns: (The created/retrieved Track object, created) """ - track_obj_query = Track.objects.filter(track_id__exact=track_dict['id']) - if len(track_obj_query) == 0: - new_track = Track.objects.create( - track_id=track_dict['id'], - year=track_dict['album']['release_date'].split('-')[0], - popularity=int(track_dict['popularity']), - runtime=int(float(track_dict['duration_ms']) / 1000), - name=track_dict['name'], - ) - # print("pop/run: ", new_track.popularity, new_track.runtime) - - # have to add artists and user after saving object since track needs to - # have ID before filling in m2m field + new_track, created = Track.objects.get_or_create( + track_id=track_dict['id'], + year=track_dict['album']['release_date'].split('-')[0], + popularity=int(track_dict['popularity']), + runtime=int(float(track_dict['duration_ms']) / 1000), + name=track_dict['name'], + ) + + # have to add artists and user after saving object since track needs to + # have ID before filling in m2m field + if created: for artist in artists: new_track.artists.add(artist) new_track.users.add(user) new_track.save() - return new_track - elif len(track_obj_query) == 1: - return track_obj_query[0] + return new_track, created # }}} save_track_obj # # get_audio_features {{{ # -def get_audio_features(headers, track_id, track): - """Returns the audio features of a soundtrack +def save_audio_features(headers, track_id, track): + """Creates and saves a new AudioFeatures object Args: headers: headers containing the API token track_id: the id of the soundtrack, needed to query the Spotify API - track: Track object to associate with the AudioFeatures object + track: Track object to associate with the new AudioFeatures object - Returns: - A dictionary with the features as its keys, if audio feature data is missing for the track, - an empty dictionary is returned. """ response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json() @@ -141,7 +138,6 @@ def get_audio_features(headers, track_id, track): setattr(audio_features_entry, key, val) audio_features_entry.save() - return features_dict # }}} get_audio_features # @@ -300,30 +296,27 @@ def get_track_info(track_dict, library_stats, sample_size): # }}} get_track_info # -# calculate_genres_from_artists {{{ # +# update_genres_from_artists {{{ # + -def calculate_genres_from_artists(headers, library_stats): - """Tallies up genre counts based on artists in library_stats. +def update_artist_genre(headers, artist_obj): + """Updates the top genre for an artist by querying the Spotify API :headers: For making the API call. - :library_stats: Dictionary containing the data mined from user's Spotify library + :artist_obj: the Artist object whose genre field will be updated :returns: None """ - for artist_entry in library_stats['artists'].values(): - artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_entry['id'], headers=headers).json() - # increase each genre count by artist count - for genre in artist_response['genres']: - increase_nested_key('genres', genre, library_stats, artist_entry['count']) - - # update genre for artist in database with top genre - Artist.objects.filter(artist_id=artist_entry['id']).update(genre=artist_response['genres'][0]) + artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_obj.id, headers=headers).json() + # update genre for artist in database with top genre + artist_obj.update(genre=artist_response['genres'][0]) # }}} calculate_genres_from_artists # # process_library_stats {{{ # + def process_library_stats(library_stats): """Processes library_stats into format more suitable for D3 consumption @@ -388,23 +381,3 @@ def get_genre_data(user): # user_tracks = Track.objects.filter(users__exact=user) # for track in user_tracks: # print(track.name) - - -def get_artist_data(user_id): - """Return artist data needed to create the graph for user. - - :user_id: user ID for which to return the data for. - :returns: List of dicts containing counts for each artist. - - """ - # TODO: not actual artists for user - # PICK UP: figure out how to pass data to D3/frontend - print(user_id) - # user = User.objects.get(user_id=user_id) - artist_counts = Artist.objects.annotate(num_songs=Count('track')) - processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] - # for artist in artist_counts: - # print(artist.name, artist.num_songs) - return JsonResponse(processed_artist_data, safe=False) - # return serializers.serialize('json', processed_artist_data) - # return processed_artist_data diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 3ab4f61..6bf090b 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -10,9 +10,9 @@ import pprint from datetime import datetime from django.shortcuts import render, redirect -from django.http import HttpResponse, HttpResponseBadRequest +from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.db.models import Count -from .utils import parse_library, process_library_stats, get_artist_data +from .utils import parse_library, process_library_stats from .models import User, Track, AudioFeatures, Artist # }}} imports # @@ -44,6 +44,7 @@ def generate_random_string(length): # token_expired {{{ # + def token_expired(token_obtained_at, valid_for): """Returns True if token expired, False if otherwise @@ -143,6 +144,8 @@ def user_data(request): user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json() request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model # request.session['user_name'] = user_data_response['display_name'] + + # get_or_create() returns a tuple (obj, created) user = User.objects.get_or_create(user_id=user_data_response['id'])[0] context = { @@ -174,8 +177,20 @@ def user_data(request): def test_db(request): user_id = "polarbier" context = { - 'artist_data': get_artist_data(user_id), 'user_id': user_id, } # get_artist_data(user) return render(request, 'spotifyvis/test_db.html', context) + + +def get_artist_data(request, user_id): + + # TODO: not actual artists for user + # PICK UP: figure out how to pass data to D3/frontend + print(user_id) + # user = User.objects.get(user_id=user_id) + artist_counts = Artist.objects.annotate(num_songs=Count('track')) + processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] + # for artist in artist_counts: + # print(artist.name, artist.num_songs) + return JsonResponse(data=processed_artist_data, safe=False)