From e07fd0965a713653133dcc8261648f10a3f59e1e Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Mon, 4 Jun 2018 17:21:18 -0400 Subject: [PATCH 01/49] Added Vim folds to models.py and utils.py --- spotifyvis/models.py | 13 ++++++++++++- spotifyvis/utils.py | 22 ++++++++++++++++++++++ spotifyvis/views.py | 6 +++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 8d28fda..f5e28c4 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -1,5 +1,7 @@ from django.db import models +# Artist {{{ # + class Artist(models.Model): class Meta: @@ -14,6 +16,9 @@ class Artist(models.Model): def __str__(self): return self.name +# }}} Artist # + +# User {{{ # class User(models.Model): @@ -27,6 +32,9 @@ class User(models.Model): def __str__(self): return self.username +# }}} User # + +# Track {{{ # class Track(models.Model): @@ -46,6 +54,9 @@ class Track(models.Model): def __str__(self): return self.name +# }}} Track # + +# AudioFeatures {{{ # class AudioFeatures(models.Model): @@ -66,4 +77,4 @@ class AudioFeatures(models.Model): def __str__(self): return super(AudioFeatures, self).__str__() - +# }}} AudioFeatures # diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 9e537dc..4702a94 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -1,6 +1,11 @@ +# imports {{{ # + import requests import math import pprint +from models import * + +# }}} imports # # parse_library {{{ # @@ -44,6 +49,8 @@ def parse_library(headers, tracks, library_stats): # }}} parse_library # +# get_audio_features {{{ # + def get_audio_features(headers, track_id): """Returns the audio features of a soundtrack @@ -71,6 +78,9 @@ def get_audio_features(headers, track_id): return features_dict +# }}} get_audio_features # + +# update_std_dev {{{ # def update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size): """Calculates the standard deviation for a sample without storing all data points @@ -94,6 +104,9 @@ def update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size): )) return new_mean, new_std_dev +# }}} update_std_dev # + +# update_audio_feature_stats {{{ # def update_audio_feature_stats(feature, new_data_point, sample_size, library_stats): """Updates the audio feature statistics in library_stats @@ -124,6 +137,7 @@ def update_audio_feature_stats(feature, new_data_point, sample_size, library_sta "std_dev": new_std_dev } +# }}} update_audio_feature_stats # # increase_nested_key {{{ # @@ -167,6 +181,8 @@ def increase_artist_count(headers, artist_name, artist_id, library_stats): # }}} increase_artist_count # +# update_popularity_stats {{{ # + def update_popularity_stats(new_data_point, library_stats, sample_size): """Updates the popularity statistics in library_stats @@ -193,6 +209,8 @@ def update_popularity_stats(new_data_point, library_stats, sample_size): "std_dev": new_std_dev, } +# }}} update_popularity_stats # + # get_track_info {{{ # def get_track_info(track_dict, library_stats, sample_size): @@ -241,6 +259,8 @@ def calculate_genres_from_artists(headers, library_stats): # }}} calculate_genres_from_artists # +# process_library_stats {{{ # + def process_library_stats(library_stats): """Processes library_stats into format more suitable for D3 consumption @@ -291,3 +311,5 @@ def process_library_stats(library_stats): processed_library_stats[key] = library_stats[key] return processed_library_stats + +# }}} process_library_stats # diff --git a/spotifyvis/views.py b/spotifyvis/views.py index cfdf114..c391516 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -1,3 +1,5 @@ +# imports {{{ # + from django.shortcuts import render, redirect from django.http import HttpResponse, HttpResponseBadRequest import math @@ -10,6 +12,8 @@ import pprint from datetime import datetime from .utils import parse_library, process_library_stats +# }}} imports # + TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' library_stats = {"audio_features":{}, "genres":{}, "year_released":{}, "artists":{}, "num_songs":0, "popularity":[], "total_runtime":0} @@ -158,4 +162,4 @@ def user_data(request): pprint.pprint(processed_library_stats) return render(request, 'spotifyvis/user_data.html', context) -# }}} user_data # \ No newline at end of file +# }}} user_data # From 549af96db755b1174f1aa82841ef9c4fe497f83b Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Mon, 4 Jun 2018 21:52:30 -0400 Subject: [PATCH 02/49] Setup Artist table Removed printing of library_stats. --- .gitignore | 5 ++--- spotifyvis/utils.py | 18 ++++++++++++------ spotifyvis/views.py | 6 +++--- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 941775f..0c129a0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,5 @@ db.sqlite3 api-keys.sh Pipfile -super-pass.txt -*.js -*.ini +*.txt +graph.js diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 929ff76..9fc0dc7 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -51,7 +51,7 @@ def parse_library(headers, tracks, library_stats, user): library_stats['num_songs'] = offset + len(saved_tracks_response['items']) offset += limit calculate_genres_from_artists(headers, library_stats) - pprint.pprint(library_stats) + # pprint.pprint(library_stats) # }}} parse_library # @@ -185,6 +185,14 @@ def increase_artist_count(headers, artist_name, artist_id, library_stats): else: library_stats['artists'][artist_name]['count'] += 1 + # add artist to database if new + if len(Artist.objects.filter(artist_id__contains=artist_id)) == 0: + new_artist = Artist( + artist_id=artist_id, + name=artist_name, + ) + new_artist.save() + # }}} increase_artist_count # # update_popularity_stats {{{ # @@ -236,11 +244,6 @@ def get_track_info(track_dict, library_stats, sample_size): year_released = track_dict['album']['release_date'].split('-')[0] increase_nested_key('year_released', year_released, library_stats) - # artist - # artist_names = [artist['name'] for artist in track_dict['artists']] - # for artist_name in artist_names: - # increase_nested_key('artists', artist_name) - # runtime library_stats['total_runtime'] += float(track_dict['duration_ms']) / (1000 * 60) @@ -263,6 +266,9 @@ def calculate_genres_from_artists(headers, library_stats): for genre in artist_response['genres']: increase_nested_key('genres', genre, library_stats, artist_entry['count']) + # update genre for artist in database with top genre + Artist.objects.filter(artist_id=artist_entry['id']).update(genre=artist_response['genres'][0]) + # }}} calculate_genres_from_artists # # process_library_stats {{{ # diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 8e9c7b8..d6800bd 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -164,9 +164,9 @@ def user_data(request): } parse_library(headers, TRACKS_TO_QUERY, library_stats, user) processed_library_stats = process_library_stats(library_stats) - print("================================================") - print("Processed data follows\n") - pprint.pprint(processed_library_stats) + # print("================================================") + # print("Processed data follows\n") + # pprint.pprint(processed_library_stats) return render(request, 'spotifyvis/user_data.html', context) # }}} user_data # From b73588789909110dd6068b603dbc8373e8ac23eb Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Mon, 4 Jun 2018 22:17:31 -0400 Subject: [PATCH 03/49] Bug fixes Server crashes as of the last commit, now fixed. --- .../migrations/0002_auto_20180605_0209.py | 18 ++++++++++++++++++ .../migrations/0003_auto_20180605_0211.py | 18 ++++++++++++++++++ spotifyvis/models.py | 2 +- spotifyvis/templates/spotifyvis/user_data.html | 1 + spotifyvis/views.py | 15 ++++++++++----- 5 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 spotifyvis/migrations/0002_auto_20180605_0209.py create mode 100644 spotifyvis/migrations/0003_auto_20180605_0211.py diff --git a/spotifyvis/migrations/0002_auto_20180605_0209.py b/spotifyvis/migrations/0002_auto_20180605_0209.py new file mode 100644 index 0000000..fe5fb37 --- /dev/null +++ b/spotifyvis/migrations/0002_auto_20180605_0209.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.5 on 2018-06-05 02:09 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('spotifyvis', '0001_initial'), + ] + + operations = [ + migrations.RenameField( + model_name='user', + old_name='username', + new_name='user_name', + ), + ] diff --git a/spotifyvis/migrations/0003_auto_20180605_0211.py b/spotifyvis/migrations/0003_auto_20180605_0211.py new file mode 100644 index 0000000..9e59bb3 --- /dev/null +++ b/spotifyvis/migrations/0003_auto_20180605_0211.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.5 on 2018-06-05 02:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('spotifyvis', '0002_auto_20180605_0209'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='user_name', + field=models.CharField(blank=True, max_length=30), + ), + ] diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 773492c..02a081d 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -21,7 +21,7 @@ class User(models.Model): verbose_name_plural = "Users" user_id = models.CharField(primary_key=True, max_length=30) # the user's Spotify ID - username = models.CharField(max_length=30) # User's Spotify user name, if set + user_name = models.CharField(max_length=30, blank=True) # User's Spotify user name, if set def __str__(self): return self.username diff --git a/spotifyvis/templates/spotifyvis/user_data.html b/spotifyvis/templates/spotifyvis/user_data.html index c008bdd..209350a 100644 --- a/spotifyvis/templates/spotifyvis/user_data.html +++ b/spotifyvis/templates/spotifyvis/user_data.html @@ -1,3 +1,4 @@ +{% load static %} diff --git a/spotifyvis/views.py b/spotifyvis/views.py index bd7a300..9a9fdd1 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -135,16 +135,21 @@ def user_data(request): user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json() request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model - request.session['user_name'] = user_data_response['display_name'] + display_name = user_data_response['display_name'] + if display_name is not None: + request.session['user_name'] = display_name + else: + request.session['user_name'] = "" user = None # will be set to the current user object later try: user = User.objects.get(user_id=request.session['user_id']) except User.DoesNotExist: user = User.objects.create(user_id=request.session['user_id'], user_name=request.session['user_name']) - # context = { - # 'user_name': user_data_response['display_name'], - # 'id': user_data_response['id'], - # } + + context = { + 'user_name': user_data_response['display_name'], + 'id': user_data_response['id'], + } library_stats = { "audio_features":{}, From 27718743d74af36d9daca6db5c83749ee0e619f3 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Tue, 5 Jun 2018 19:55:36 -0400 Subject: [PATCH 04/49] Change artists field in Track artists is now a ManyToManyField in Track, which is cleaner than using a unique_together clause in the Meta class. --- spotifyvis/migrations/0001_initial.py | 10 +++------- ...0605_0209.py => 0002_auto_20180605_2347.py} | 8 ++++---- .../migrations/0003_auto_20180605_0211.py | 18 ------------------ spotifyvis/models.py | 9 ++++----- spotifyvis/views.py | 4 +++- 5 files changed, 14 insertions(+), 35 deletions(-) rename spotifyvis/migrations/{0002_auto_20180605_0209.py => 0002_auto_20180605_2347.py} (59%) delete mode 100644 spotifyvis/migrations/0003_auto_20180605_0211.py diff --git a/spotifyvis/migrations/0001_initial.py b/spotifyvis/migrations/0001_initial.py index e22ffd2..18f13b6 100644 --- a/spotifyvis/migrations/0001_initial.py +++ b/spotifyvis/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.0.5 on 2018-06-03 23:01 +# Generated by Django 2.0.5 on 2018-06-05 20:53 from django.db import migrations, models import django.db.models.deletion @@ -43,7 +43,7 @@ class Migration(migrations.Migration): name='User', fields=[ ('user_id', models.CharField(max_length=30, primary_key=True, serialize=False)), - ('username', models.CharField(max_length=30)), + ('user_name', models.CharField(blank=True, max_length=30)), ], options={ 'verbose_name': 'User', @@ -71,15 +71,11 @@ class Migration(migrations.Migration): migrations.AddField( model_name='track', name='artist', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='spotifyvis.Artist'), + field=models.ManyToManyField(to='spotifyvis.Artist'), ), migrations.AddField( model_name='track', name='users', field=models.ManyToManyField(to='spotifyvis.User'), ), - migrations.AlterUniqueTogether( - name='track', - unique_together={('track_id', 'artist')}, - ), ] diff --git a/spotifyvis/migrations/0002_auto_20180605_0209.py b/spotifyvis/migrations/0002_auto_20180605_2347.py similarity index 59% rename from spotifyvis/migrations/0002_auto_20180605_0209.py rename to spotifyvis/migrations/0002_auto_20180605_2347.py index fe5fb37..af053ab 100644 --- a/spotifyvis/migrations/0002_auto_20180605_0209.py +++ b/spotifyvis/migrations/0002_auto_20180605_2347.py @@ -1,4 +1,4 @@ -# Generated by Django 2.0.5 on 2018-06-05 02:09 +# Generated by Django 2.0.5 on 2018-06-05 23:47 from django.db import migrations @@ -11,8 +11,8 @@ class Migration(migrations.Migration): operations = [ migrations.RenameField( - model_name='user', - old_name='username', - new_name='user_name', + model_name='track', + old_name='artist', + new_name='artists', ), ] diff --git a/spotifyvis/migrations/0003_auto_20180605_0211.py b/spotifyvis/migrations/0003_auto_20180605_0211.py deleted file mode 100644 index 9e59bb3..0000000 --- a/spotifyvis/migrations/0003_auto_20180605_0211.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.0.5 on 2018-06-05 02:11 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('spotifyvis', '0002_auto_20180605_0209'), - ] - - operations = [ - migrations.AlterField( - model_name='user', - name='user_name', - field=models.CharField(blank=True, max_length=30), - ), - ] diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 02a081d..7fc5767 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -20,11 +20,11 @@ class User(models.Model): verbose_name = "User" verbose_name_plural = "Users" - user_id = models.CharField(primary_key=True, max_length=30) # the user's Spotify ID - user_name = models.CharField(max_length=30, blank=True) # User's Spotify user name, if set + user_id = models.CharField(primary_key=True, max_length=30) # the user's Spotify ID + user_name = models.CharField(max_length=30, blank=True) # User's Spotify user name, if set def __str__(self): - return self.username + return self.user_name class Track(models.Model): @@ -32,10 +32,9 @@ class Track(models.Model): class Meta: verbose_name = "Track" verbose_name_plural = "Tracks" - unique_together = ('track_id', 'artist',) track_id = models.CharField(max_length=30) - artist = models.ForeignKey(Artist, on_delete=models.CASCADE) + artists = models.ManyToManyField(Artist) year = models.PositiveSmallIntegerField() popularity = models.DecimalField(decimal_places=2, max_digits=2) runtime = models.PositiveSmallIntegerField() diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 9a9fdd1..55ee2a3 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -16,6 +16,7 @@ TRACKS_TO_QUERY = 5 # generate_random_string {{{ # + def generate_random_string(length): """Generates a random string of a certain length @@ -145,7 +146,8 @@ def user_data(request): user = User.objects.get(user_id=request.session['user_id']) except User.DoesNotExist: user = User.objects.create(user_id=request.session['user_id'], user_name=request.session['user_name']) - + user.save() + context = { 'user_name': user_data_response['display_name'], 'id': user_data_response['id'], From cdcc33c0141a8a1b5013119d0ae261c5f6f54375 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 6 Jun 2018 03:46:09 -0400 Subject: [PATCH 05/49] Setup Track table, modified models to data Recreated migrations from scratch to fix bug with initializing Track objects. --- musicvis/settings.py | 2 +- spotifyvis/migrations/0001_initial.py | 26 +++++------- spotifyvis/models.py | 22 ++++++---- spotifyvis/utils.py | 59 +++++++++++++++++++++------ spotifyvis/views.py | 9 ++-- 5 files changed, 73 insertions(+), 45 deletions(-) diff --git a/musicvis/settings.py b/musicvis/settings.py index 7eefdbb..0cedb25 100644 --- a/musicvis/settings.py +++ b/musicvis/settings.py @@ -110,7 +110,7 @@ AUTH_PASSWORD_VALIDATORS = [ LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +TIME_ZONE = 'America/Toronto' USE_I18N = True diff --git a/spotifyvis/migrations/0001_initial.py b/spotifyvis/migrations/0001_initial.py index e22ffd2..167e531 100644 --- a/spotifyvis/migrations/0001_initial.py +++ b/spotifyvis/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.0.5 on 2018-06-03 23:01 +# Generated by Django 2.0.5 on 2018-06-06 07:26 from django.db import migrations, models import django.db.models.deletion @@ -20,34 +20,32 @@ class Migration(migrations.Migration): ('genre', models.CharField(max_length=20)), ], options={ - 'verbose_name': 'Artist', 'verbose_name_plural': 'Artists', + 'verbose_name': 'Artist', }, ), migrations.CreateModel( name='Track', fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('track_id', models.CharField(max_length=30)), + ('track_id', models.CharField(max_length=30, primary_key=True, serialize=False)), ('year', models.PositiveSmallIntegerField()), - ('popularity', models.DecimalField(decimal_places=2, max_digits=2)), + ('popularity', models.PositiveSmallIntegerField()), ('runtime', models.PositiveSmallIntegerField()), ('name', models.CharField(max_length=75)), ], options={ - 'verbose_name': 'Track', 'verbose_name_plural': 'Tracks', + 'verbose_name': 'Track', }, ), migrations.CreateModel( name='User', fields=[ ('user_id', models.CharField(max_length=30, primary_key=True, serialize=False)), - ('username', models.CharField(max_length=30)), ], options={ - 'verbose_name': 'User', 'verbose_name_plural': 'Users', + 'verbose_name': 'User', }, ), migrations.CreateModel( @@ -64,22 +62,18 @@ class Migration(migrations.Migration): ('tempo', models.DecimalField(decimal_places=2, max_digits=2)), ], options={ - 'verbose_name': 'AudioFeatures', 'verbose_name_plural': 'AudioFeatures', + 'verbose_name': 'AudioFeatures', }, ), migrations.AddField( model_name='track', - name='artist', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='spotifyvis.Artist'), + name='artists', + field=models.ManyToManyField(blank=True, to='spotifyvis.Artist'), ), migrations.AddField( model_name='track', name='users', - field=models.ManyToManyField(to='spotifyvis.User'), - ), - migrations.AlterUniqueTogether( - name='track', - unique_together={('track_id', 'artist')}, + field=models.ManyToManyField(blank=True, to='spotifyvis.User'), ), ] diff --git a/spotifyvis/models.py b/spotifyvis/models.py index d33da87..15fee46 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -1,5 +1,8 @@ from django.db import models +# id's are 22 in length in examples but set to 30 for buffer +id_length=30 + # Artist {{{ # class Artist(models.Model): @@ -7,7 +10,7 @@ class Artist(models.Model): verbose_name = "Artist" verbose_name_plural = "Artists" - artist_id = models.CharField(primary_key=True, max_length=30) + artist_id = models.CharField(primary_key=True, max_length=id_length) # unique since only storing one genre per artist right now name = models.CharField(unique=True, max_length=50) genre = models.CharField(max_length=20) @@ -24,11 +27,11 @@ class User(models.Model): verbose_name = "User" verbose_name_plural = "Users" - user_id = models.CharField(primary_key=True, max_length=30) # the user's Spotify ID - username = models.CharField(max_length=30) # User's Spotify user name, if set + user_id = models.CharField(primary_key=True, max_length=id_length) # the user's Spotify ID + # username = models.CharField(max_length=30) # User's Spotify user name, if set def __str__(self): - return self.username + return self.user_id # }}} User # @@ -39,15 +42,16 @@ class Track(models.Model): class Meta: verbose_name = "Track" verbose_name_plural = "Tracks" - unique_together = ('track_id', 'artist',) + # unique_together = ('track_id', 'artist',) - track_id = models.CharField(max_length=30) - artist = models.ForeignKey(Artist, on_delete=models.CASCADE) + track_id = models.CharField(primary_key=True, max_length=id_length) + # artist = models.ForeignKey(Artist, on_delete=models.CASCADE) + artists = models.ManyToManyField(Artist, blank=True) year = models.PositiveSmallIntegerField() - popularity = models.DecimalField(decimal_places=2, max_digits=2) + popularity = models.PositiveSmallIntegerField() runtime = models.PositiveSmallIntegerField() name = models.CharField(max_length=75) - users = models.ManyToManyField(User) + users = models.ManyToManyField(User, blank=True) def __str__(self): return self.name diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 9fc0dc7..cc8ff7c 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -30,23 +30,38 @@ def parse_library(headers, tracks, library_stats, user): num_samples = 0 # number of actual track samples feature_data_points = 0 # number of feature data analyses (some tracks do not have analyses available) + # iterate until hit requested num of tracks for _ in range(0, tracks, limit): payload['offset'] = str(offset) + # get current set of tracks saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', headers=headers, params=payload).json() + + # TODO: refactor the for loop body into helper function + # iterate through each track for track_dict in saved_tracks_response['items']: num_samples += 1 + # update artist info before track so that Track object can reference + # Artist object + track_artists = [] + for artist_dict in track_dict['track']['artists']: + increase_artist_count(headers, artist_dict['name'], + artist_dict['id'], library_stats) + track_artists.append(Artist.objects.get_or_create( + artist_id=artist_dict['id'], + name=artist_dict['name'], + )[0]) + + save_track_obj(track_dict['track'], track_artists, user) get_track_info(track_dict['track'], library_stats, num_samples) - # get_genre(headers, track_dict['track']['album']['id']) audio_features_dict = get_audio_features(headers, track_dict['track']['id']) if len(audio_features_dict) != 0: # Track the number of audio analyses for calculating # audio feature averages and standard deviations on the fly feature_data_points += 1 - for feature, feature_data in audio_features_dict.items(): - update_audio_feature_stats(feature, feature_data, feature_data_points, library_stats) - for artist_dict in track_dict['track']['artists']: - increase_artist_count(headers, artist_dict['name'], artist_dict['id'], library_stats) + update_audio_feature_stats(feature, feature_data, + feature_data_points, library_stats) + # calculates num_songs with offset + songs retrieved library_stats['num_songs'] = offset + len(saved_tracks_response['items']) offset += limit @@ -55,6 +70,32 @@ def parse_library(headers, tracks, library_stats, user): # }}} parse_library # +def save_track_obj(track_dict, artists, user): + """Make an entry in the database for this track if it doesn't exist already. + + :track_dict: TODO + :artists: artists of the song, passed in as a list of Artist objects. + :user: TODO + :returns: None + + """ + if len(Track.objects.filter(track_id__exact=track_dict['id'])) == 0: + new_track = Track.objects.create( + track_id=track_dict['id'], + year=track_dict['album']['release_date'].split('-')[0], + popularity=int(track_dict['popularity']), + runtime=int(float(track_dict['duration_ms']) / 1000), + name=track_dict['name'], + ) + # print("pop/run: ", new_track.popularity, new_track.runtime) + + # have to add artists and user after saving object since track needs to + # have ID before filling in m2m field + for artist in artists: + new_track.artists.add(artist) + new_track.users.add(user) + new_track.save() + # get_audio_features {{{ # def get_audio_features(headers, track_id): @@ -185,14 +226,6 @@ def increase_artist_count(headers, artist_name, artist_id, library_stats): else: library_stats['artists'][artist_name]['count'] += 1 - # add artist to database if new - if len(Artist.objects.filter(artist_id__contains=artist_id)) == 0: - new_artist = Artist( - artist_id=artist_id, - name=artist_name, - ) - new_artist.save() - # }}} increase_artist_count # # update_popularity_stats {{{ # diff --git a/spotifyvis/views.py b/spotifyvis/views.py index d6800bd..65c9a31 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -139,12 +139,9 @@ def user_data(request): user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json() request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model - request.session['user_name'] = user_data_response['display_name'] - user = None # will be set to the current user object later - # try: - # user = User.objects.get(user_id=request.session['user_id']) - # except User.DoesNotExist: - # user = User.objects.create(user_id=request.session['user_id'], user_name=request.session['user_name']) + # request.session['user_name'] = user_data_response['display_name'] + user = User.objects.get_or_create(user_id=user_data_response['id'])[0] + context = { 'user_name': user_data_response['display_name'], 'id': user_data_response['id'], From 62cc6c8ccd6178a764d6a309fee1d2cebcea70f6 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 6 Jun 2018 05:36:00 -0400 Subject: [PATCH 06/49] Setup AudioFeatures table --- .../migrations/0002_auto_20180606_0523.py | 53 +++++++++++++++++++ .../migrations/0003_auto_20180606_0525.py | 23 ++++++++ spotifyvis/models.py | 16 +++--- spotifyvis/utils.py | 28 +++++++--- 4 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 spotifyvis/migrations/0002_auto_20180606_0523.py create mode 100644 spotifyvis/migrations/0003_auto_20180606_0525.py diff --git a/spotifyvis/migrations/0002_auto_20180606_0523.py b/spotifyvis/migrations/0002_auto_20180606_0523.py new file mode 100644 index 0000000..7aaa661 --- /dev/null +++ b/spotifyvis/migrations/0002_auto_20180606_0523.py @@ -0,0 +1,53 @@ +# Generated by Django 2.0.5 on 2018-06-06 09:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('spotifyvis', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='audiofeatures', + name='acousticness', + field=models.DecimalField(decimal_places=3, max_digits=3), + ), + migrations.AlterField( + model_name='audiofeatures', + name='danceability', + field=models.DecimalField(decimal_places=3, max_digits=3), + ), + migrations.AlterField( + model_name='audiofeatures', + name='energy', + field=models.DecimalField(decimal_places=3, max_digits=3), + ), + migrations.AlterField( + model_name='audiofeatures', + name='instrumentalness', + field=models.DecimalField(decimal_places=3, max_digits=3), + ), + migrations.AlterField( + model_name='audiofeatures', + name='loudness', + field=models.DecimalField(decimal_places=3, max_digits=3), + ), + migrations.AlterField( + model_name='audiofeatures', + name='speechiness', + field=models.DecimalField(decimal_places=3, max_digits=3), + ), + migrations.AlterField( + model_name='audiofeatures', + name='tempo', + field=models.DecimalField(decimal_places=3, max_digits=3), + ), + migrations.AlterField( + model_name='audiofeatures', + name='valence', + field=models.DecimalField(decimal_places=3, max_digits=3), + ), + ] diff --git a/spotifyvis/migrations/0003_auto_20180606_0525.py b/spotifyvis/migrations/0003_auto_20180606_0525.py new file mode 100644 index 0000000..b8b50db --- /dev/null +++ b/spotifyvis/migrations/0003_auto_20180606_0525.py @@ -0,0 +1,23 @@ +# Generated by Django 2.0.5 on 2018-06-06 09:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('spotifyvis', '0002_auto_20180606_0523'), + ] + + operations = [ + migrations.AlterField( + model_name='audiofeatures', + name='loudness', + field=models.DecimalField(decimal_places=3, max_digits=6), + ), + migrations.AlterField( + model_name='audiofeatures', + name='tempo', + field=models.DecimalField(decimal_places=3, max_digits=6), + ), + ] diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 15fee46..50be45f 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -67,14 +67,14 @@ class AudioFeatures(models.Model): verbose_name_plural = "AudioFeatures" track = models.OneToOneField(Track, on_delete=models.CASCADE, primary_key=True,) - danceability = models.DecimalField(decimal_places=2, max_digits=2) - energy = models.DecimalField(decimal_places=2, max_digits=2) - loudness = models.DecimalField(decimal_places=2, max_digits=2) - speechiness = models.DecimalField(decimal_places=2, max_digits=2) - acousticness = models.DecimalField(decimal_places=2, max_digits=2) - instrumentalness = models.DecimalField(decimal_places=2, max_digits=2) - valence = models.DecimalField(decimal_places=2, max_digits=2) - tempo = models.DecimalField(decimal_places=2, max_digits=2) + acousticness = models.DecimalField(decimal_places=3, max_digits=3) + danceability = models.DecimalField(decimal_places=3, max_digits=3) + energy = models.DecimalField(decimal_places=3, max_digits=3) + instrumentalness = models.DecimalField(decimal_places=3, max_digits=3) + loudness = models.DecimalField(decimal_places=3, max_digits=6) + speechiness = models.DecimalField(decimal_places=3, max_digits=3) + tempo = models.DecimalField(decimal_places=3, max_digits=6) + valence = models.DecimalField(decimal_places=3, max_digits=3) def __str__(self): return super(AudioFeatures, self).__str__() diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index cc8ff7c..8ed1cc6 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -51,9 +51,10 @@ def parse_library(headers, tracks, library_stats, user): name=artist_dict['name'], )[0]) - save_track_obj(track_dict['track'], track_artists, user) + track_obj = save_track_obj(track_dict['track'], track_artists, user) get_track_info(track_dict['track'], library_stats, num_samples) - audio_features_dict = get_audio_features(headers, track_dict['track']['id']) + audio_features_dict = get_audio_features(headers, + track_dict['track']['id'], track_obj) if len(audio_features_dict) != 0: # Track the number of audio analyses for calculating # audio feature averages and standard deviations on the fly @@ -70,16 +71,19 @@ def parse_library(headers, tracks, library_stats, user): # }}} parse_library # +# save_track_obj {{{ # + def save_track_obj(track_dict, artists, user): """Make an entry in the database for this track if it doesn't exist already. - :track_dict: TODO + :track_dict: dictionary from the API call containing track information. :artists: artists of the song, passed in as a list of Artist objects. - :user: TODO - :returns: None + :user: User object for which this Track is to be associated with. + :returns: The created/retrieved Track object. """ - if len(Track.objects.filter(track_id__exact=track_dict['id'])) == 0: + track_obj_query = Track.objects.filter(track_id__exact=track_dict['id']) + if len(track_obj_query) == 0: new_track = Track.objects.create( track_id=track_dict['id'], year=track_dict['album']['release_date'].split('-')[0], @@ -95,15 +99,21 @@ def save_track_obj(track_dict, artists, user): new_track.artists.add(artist) new_track.users.add(user) new_track.save() + return new_track + elif len(track_obj_query) == 1: + return track_obj_query[0] + +# }}} save_track_obj # # get_audio_features {{{ # -def get_audio_features(headers, track_id): +def get_audio_features(headers, track_id, track): """Returns the audio features of a soundtrack Args: headers: headers containing the API token track_id: the id of the soundtrack, needed to query the Spotify API + track: Track object to associate with the AudioFeatures object Returns: A dictionary with the features as its keys, if audio feature data is missing for the track, @@ -119,9 +129,13 @@ def get_audio_features(headers, track_id): useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", ] + audio_features_entry = AudioFeatures() + audio_features_entry.track = track for key, val in response.items(): if key not in useless_keys: features_dict[key] = val + setattr(audio_features_entry, key, val) + audio_features_entry.save() return features_dict From b66cf1dd87a3f67874aefe411beceae724f39557 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Wed, 6 Jun 2018 20:56:40 -0400 Subject: [PATCH 07/49] Delete all migrations and start anew Deleted all migrations and started anew to incorporate changes in models. --- spotifyvis/migrations/0001_initial.py | 26 ++++----- .../migrations/0002_auto_20180605_2347.py | 18 ------- .../migrations/0002_auto_20180606_0523.py | 53 ------------------- .../migrations/0003_auto_20180606_0525.py | 23 -------- 4 files changed, 14 insertions(+), 106 deletions(-) delete mode 100644 spotifyvis/migrations/0002_auto_20180605_2347.py delete mode 100644 spotifyvis/migrations/0002_auto_20180606_0523.py delete mode 100644 spotifyvis/migrations/0003_auto_20180606_0525.py diff --git a/spotifyvis/migrations/0001_initial.py b/spotifyvis/migrations/0001_initial.py index db17f34..fb92c44 100644 --- a/spotifyvis/migrations/0001_initial.py +++ b/spotifyvis/migrations/0001_initial.py @@ -1,3 +1,5 @@ +# Generated by Django 2.0.5 on 2018-06-07 00:52 + from django.db import migrations, models import django.db.models.deletion @@ -18,8 +20,8 @@ class Migration(migrations.Migration): ('genre', models.CharField(max_length=20)), ], options={ - 'verbose_name_plural': 'Artists', 'verbose_name': 'Artist', + 'verbose_name_plural': 'Artists', }, ), migrations.CreateModel( @@ -32,8 +34,8 @@ class Migration(migrations.Migration): ('name', models.CharField(max_length=75)), ], options={ - 'verbose_name_plural': 'Tracks', 'verbose_name': 'Track', + 'verbose_name_plural': 'Tracks', }, ), migrations.CreateModel( @@ -42,26 +44,26 @@ class Migration(migrations.Migration): ('user_id', models.CharField(max_length=30, primary_key=True, serialize=False)), ], options={ - 'verbose_name_plural': 'Users', 'verbose_name': 'User', + 'verbose_name_plural': 'Users', }, ), migrations.CreateModel( name='AudioFeatures', fields=[ ('track', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='spotifyvis.Track')), - ('danceability', models.DecimalField(decimal_places=2, max_digits=2)), - ('energy', models.DecimalField(decimal_places=2, max_digits=2)), - ('loudness', models.DecimalField(decimal_places=2, max_digits=2)), - ('speechiness', models.DecimalField(decimal_places=2, max_digits=2)), - ('acousticness', models.DecimalField(decimal_places=2, max_digits=2)), - ('instrumentalness', models.DecimalField(decimal_places=2, max_digits=2)), - ('valence', models.DecimalField(decimal_places=2, max_digits=2)), - ('tempo', models.DecimalField(decimal_places=2, max_digits=2)), + ('acousticness', models.DecimalField(decimal_places=3, max_digits=3)), + ('danceability', models.DecimalField(decimal_places=3, max_digits=3)), + ('energy', models.DecimalField(decimal_places=3, max_digits=3)), + ('instrumentalness', models.DecimalField(decimal_places=3, max_digits=3)), + ('loudness', models.DecimalField(decimal_places=3, max_digits=6)), + ('speechiness', models.DecimalField(decimal_places=3, max_digits=3)), + ('tempo', models.DecimalField(decimal_places=3, max_digits=6)), + ('valence', models.DecimalField(decimal_places=3, max_digits=3)), ], options={ - 'verbose_name_plural': 'AudioFeatures', 'verbose_name': 'AudioFeatures', + 'verbose_name_plural': 'AudioFeatures', }, ), migrations.AddField( diff --git a/spotifyvis/migrations/0002_auto_20180605_2347.py b/spotifyvis/migrations/0002_auto_20180605_2347.py deleted file mode 100644 index af053ab..0000000 --- a/spotifyvis/migrations/0002_auto_20180605_2347.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.0.5 on 2018-06-05 23:47 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('spotifyvis', '0001_initial'), - ] - - operations = [ - migrations.RenameField( - model_name='track', - old_name='artist', - new_name='artists', - ), - ] diff --git a/spotifyvis/migrations/0002_auto_20180606_0523.py b/spotifyvis/migrations/0002_auto_20180606_0523.py deleted file mode 100644 index 7aaa661..0000000 --- a/spotifyvis/migrations/0002_auto_20180606_0523.py +++ /dev/null @@ -1,53 +0,0 @@ -# Generated by Django 2.0.5 on 2018-06-06 09:23 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('spotifyvis', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='audiofeatures', - name='acousticness', - field=models.DecimalField(decimal_places=3, max_digits=3), - ), - migrations.AlterField( - model_name='audiofeatures', - name='danceability', - field=models.DecimalField(decimal_places=3, max_digits=3), - ), - migrations.AlterField( - model_name='audiofeatures', - name='energy', - field=models.DecimalField(decimal_places=3, max_digits=3), - ), - migrations.AlterField( - model_name='audiofeatures', - name='instrumentalness', - field=models.DecimalField(decimal_places=3, max_digits=3), - ), - migrations.AlterField( - model_name='audiofeatures', - name='loudness', - field=models.DecimalField(decimal_places=3, max_digits=3), - ), - migrations.AlterField( - model_name='audiofeatures', - name='speechiness', - field=models.DecimalField(decimal_places=3, max_digits=3), - ), - migrations.AlterField( - model_name='audiofeatures', - name='tempo', - field=models.DecimalField(decimal_places=3, max_digits=3), - ), - migrations.AlterField( - model_name='audiofeatures', - name='valence', - field=models.DecimalField(decimal_places=3, max_digits=3), - ), - ] diff --git a/spotifyvis/migrations/0003_auto_20180606_0525.py b/spotifyvis/migrations/0003_auto_20180606_0525.py deleted file mode 100644 index b8b50db..0000000 --- a/spotifyvis/migrations/0003_auto_20180606_0525.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 2.0.5 on 2018-06-06 09:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('spotifyvis', '0002_auto_20180606_0523'), - ] - - operations = [ - migrations.AlterField( - model_name='audiofeatures', - name='loudness', - field=models.DecimalField(decimal_places=3, max_digits=6), - ), - migrations.AlterField( - model_name='audiofeatures', - name='tempo', - field=models.DecimalField(decimal_places=3, max_digits=6), - ), - ] From 79aedc655d09a0e8badefa1fadb3071174a366eb Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 7 Jun 2018 21:48:38 -0400 Subject: [PATCH 08/49] Trying to pass artist data from db to frontend --- .../static/spotifyvis/scripts/test_db.js | 11 ++++++ spotifyvis/templates/spotifyvis/index.html | 1 + spotifyvis/templates/spotifyvis/test_db.html | 27 ++++++++++++++ .../templates/spotifyvis/user_data.html | 2 +- spotifyvis/urls.py | 16 +++++---- spotifyvis/utils.py | 36 +++++++++++++++++++ spotifyvis/views.py | 17 +++++++-- 7 files changed, 100 insertions(+), 10 deletions(-) create mode 100644 spotifyvis/static/spotifyvis/scripts/test_db.js create mode 100644 spotifyvis/templates/spotifyvis/test_db.html diff --git a/spotifyvis/static/spotifyvis/scripts/test_db.js b/spotifyvis/static/spotifyvis/scripts/test_db.js new file mode 100644 index 0000000..6830820 --- /dev/null +++ b/spotifyvis/static/spotifyvis/scripts/test_db.js @@ -0,0 +1,11 @@ +console.log("{{ user_id }}"); +artist_data = JSON.parse('{{ artist_data }}'); +artist_data.forEach(function(d) { + console.log(d.name, d.num_songs); +}); + +d3.json("{% url "get_artist_data" user_id %}", function(error, data) { + data.forEach(function(d) { + console.log(d.name, d.num_songs); + }); +}); diff --git a/spotifyvis/templates/spotifyvis/index.html b/spotifyvis/templates/spotifyvis/index.html index 32d6db9..5964b34 100644 --- a/spotifyvis/templates/spotifyvis/index.html +++ b/spotifyvis/templates/spotifyvis/index.html @@ -20,6 +20,7 @@

This is an example of the Authorization Code flow

Log In (Original) + Test DB
diff --git a/spotifyvis/templates/spotifyvis/test_db.html b/spotifyvis/templates/spotifyvis/test_db.html new file mode 100644 index 0000000..3f7a0d0 --- /dev/null +++ b/spotifyvis/templates/spotifyvis/test_db.html @@ -0,0 +1,27 @@ + + + + + + + + + Test DB Page + + + + + +
    + {% for artist in artist_data %} +
  • {{ artist.name }} - {{ artist.num_songs }}
    • + {% endfor %} +
+ 
 {% filter force_escape %} {% debug %} {% endfilter %}
+ + {% load static %} + + + diff --git a/spotifyvis/templates/spotifyvis/user_data.html b/spotifyvis/templates/spotifyvis/user_data.html index f31d42f..15fb14a 100644 --- a/spotifyvis/templates/spotifyvis/user_data.html +++ b/spotifyvis/templates/spotifyvis/user_data.html @@ -20,6 +20,6 @@ {% for genre_name, genre_count in genre_dict.items %}
  • {{ genre_name }} - {{ genre_count }}
    • {% endfor %} - + diff --git a/spotifyvis/urls.py b/spotifyvis/urls.py index 0d5e894..55380bd 100644 --- a/spotifyvis/urls.py +++ b/spotifyvis/urls.py @@ -1,9 +1,13 @@ from django.urls import path, include -from . import views +from django.conf.urls import url + +from .views import * urlpatterns = [ - path('', views.index, name='index'), - path('login', views.login, name='login'), - path('callback', views.callback, name='callback'), - path('user_data', views.user_data, name='user_data'), -] \ No newline at end of file + path('', index, name='index'), + path('login', login, name='login'), + path('callback', callback, name='callback'), + path('user_data', user_data, name='user_data'), + path('test_db', test_db, name='test_db'), + path('user_artists/', get_artist_data, name='get_artist_data'), +] diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 8ed1cc6..dc684e3 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -3,7 +3,11 @@ import requests import math import pprint + from .models import Artist, User, Track, AudioFeatures +from django.db.models import Count +from django.http import JsonResponse +from django.core import serializers # }}} imports # @@ -372,3 +376,35 @@ def process_library_stats(library_stats): return processed_library_stats # }}} process_library_stats # + +def get_genre_data(user): + """Return genre data needed to create the graph user. + + :user: User object for which to return the data for. + :returns: List of dicts containing counts for each genre. + + """ + pass + # user_tracks = Track.objects.filter(users__exact=user) + # for track in user_tracks: + # print(track.name) + + +def get_artist_data(user_id): + """Return artist data needed to create the graph for user. + + :user_id: user ID for which to return the data for. + :returns: List of dicts containing counts for each artist. + + """ + # TODO: not actual artists for user + # PICK UP: figure out how to pass data to D3/frontend + print(user_id) + # user = User.objects.get(user_id=user_id) + artist_counts = Artist.objects.annotate(num_songs=Count('track')) + processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] + # for artist in artist_counts: + # print(artist.name, artist.num_songs) + return JsonResponse(processed_artist_data, safe=False) + # return serializers.serialize('json', processed_artist_data) + # return processed_artist_data diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 65c9a31..fbbe02c 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -1,7 +1,5 @@ # imports {{{ # -from django.shortcuts import render, redirect -from django.http import HttpResponse, HttpResponseBadRequest import math import random import requests @@ -10,7 +8,11 @@ import urllib import json import pprint from datetime import datetime -from .utils import parse_library, process_library_stats + +from django.shortcuts import render, redirect +from django.http import HttpResponse, HttpResponseBadRequest +from django.db.models import Count +from .utils import parse_library, process_library_stats, get_artist_data from .models import User, Track, AudioFeatures, Artist # }}} imports # @@ -167,3 +169,12 @@ def user_data(request): return render(request, 'spotifyvis/user_data.html', context) # }}} user_data # + +def test_db(request): + user_id = "polarbier" + context = { + 'artist_data': get_artist_data(user_id), + 'user_id': user_id, + } + # get_artist_data(user) + return render(request, 'spotifyvis/test_db.html', context) From 797115e9cb07b7aa9dbfe48f2e750e93fd754ab0 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 8 Jun 2018 18:45:16 -0400 Subject: [PATCH 09/49] Clean up utils.py to use database Uses of the original library_stats dictionary in utils.py were either removed or commented out, replaced by the saving of models into the database. --- spotifyvis/models.py | 45 ++++++- .../static/spotifyvis/scripts/test_db.js | 8 +- spotifyvis/templates/spotifyvis/test_db.html | 11 +- spotifyvis/utils.py | 113 +++++++----------- spotifyvis/views.py | 21 +++- 5 files changed, 110 insertions(+), 88 deletions(-) diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 19a99be..aacfd1e 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -1,8 +1,7 @@ from django.db import models # id's are 22 in length in examples but set to 30 for buffer -id_length=30 - +MAX_ID = 30 # Artist {{{ # class Artist(models.Model): @@ -10,7 +9,7 @@ class Artist(models.Model): verbose_name = "Artist" verbose_name_plural = "Artists" - artist_id = models.CharField(primary_key=True, max_length=id_length) + artist_id = models.CharField(primary_key=True, max_length=MAX_ID) # unique since only storing one genre per artist right now name = models.CharField(unique=True, max_length=50) genre = models.CharField(max_length=20) @@ -27,7 +26,7 @@ class User(models.Model): verbose_name = "User" verbose_name_plural = "Users" - user_id = models.CharField(primary_key=True, max_length=id_length) # the user's Spotify ID + user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID # username = models.CharField(max_length=30) # User's Spotify user name, if set def __str__(self): @@ -43,7 +42,7 @@ class Track(models.Model): verbose_name = "Track" verbose_name_plural = "Tracks" - track_id = models.CharField(primary_key=True, max_length=id_length) + track_id = models.CharField(primary_key=True, max_length=MAX_ID) # artist = models.ForeignKey(Artist, on_delete=models.CASCADE) artists = models.ManyToManyField(Artist, blank=True) year = models.PositiveSmallIntegerField() @@ -79,3 +78,39 @@ class AudioFeatures(models.Model): return super(AudioFeatures, self).__str__() # }}} AudioFeatures # + + +''' +class UserAudioSummary(models.Model): + """ + Stores the summary of a user's audio preferences + """ + class Meta: + verbose_name = "AudioFeatures" + verbose_name_plural = "AudioFeatures" + + user = models.OneToOneField(User, on_delete=models.CASCADE, primary_key=True,) + avg_acousticness = models.DecimalField(decimal_places=3, max_digits=3) + stdev_acousticness = models.DecimalField(decimal_places=3, max_digits=3) + + avg_danceability = models.DecimalField(decimal_places=3, max_digits=3) + stdev_danceability = models.DecimalField(decimal_places=3, max_digits=3) + + avg_energy = models.DecimalField(decimal_places=3, max_digits=3) + stdev_energy = models.DecimalField(decimal_places=3, max_digits=3) + + avg_instrumentalness = models.DecimalField(decimal_places=3, max_digits=3) + stdev_instrumentalness = models.DecimalField(decimal_places=3, max_digits=3) + + avg_loudness = models.DecimalField(decimal_places=3, max_digits=6) + stdev_loudness = models.DecimalField(decimal_places=3, max_digits=6) + + avg_speechiness = models.DecimalField(decimal_places=3, max_digits=3) + stdev_speechiness = models.DecimalField(decimal_places=3, max_digits=3) + + avg_tempo = models.DecimalField(decimal_places=3, max_digits=6) + stdev_tempo = models.DecimalField(decimal_places=3, max_digits=6) + + avg_valence = models.DecimalField(decimal_places=3, max_digits=3) + stdev_valence = models.DecimalField(decimal_places=3, max_digits=3) +''' \ No newline at end of file diff --git a/spotifyvis/static/spotifyvis/scripts/test_db.js b/spotifyvis/static/spotifyvis/scripts/test_db.js index 6830820..cb5fb67 100644 --- a/spotifyvis/static/spotifyvis/scripts/test_db.js +++ b/spotifyvis/static/spotifyvis/scripts/test_db.js @@ -1,10 +1,4 @@ -console.log("{{ user_id }}"); -artist_data = JSON.parse('{{ artist_data }}'); -artist_data.forEach(function(d) { - console.log(d.name, d.num_songs); -}); - -d3.json("{% url "get_artist_data" user_id %}", function(error, data) { +d3.json("{% url "get_artist_data", user_id %}").then(function(error, data) { data.forEach(function(d) { console.log(d.name, d.num_songs); }); diff --git a/spotifyvis/templates/spotifyvis/test_db.html b/spotifyvis/templates/spotifyvis/test_db.html index 3f7a0d0..0c51c81 100644 --- a/spotifyvis/templates/spotifyvis/test_db.html +++ b/spotifyvis/templates/spotifyvis/test_db.html @@ -20,8 +20,13 @@ {% endfor %} 
     {% filter force_escape %} {% debug %} {% endfilter %}
    - - {% load static %} - + + diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index dc684e3..1d61a9e 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -1,5 +1,4 @@ # imports {{{ # - import requests import math import pprint @@ -8,6 +7,7 @@ from .models import Artist, User, Track, AudioFeatures from django.db.models import Count from django.http import JsonResponse from django.core import serializers +import json # }}} imports # @@ -32,7 +32,6 @@ def parse_library(headers, tracks, library_stats, user): payload = {'limit': str(limit)} # use two separate variables to track, because the average popularity also requires num_samples num_samples = 0 # number of actual track samples - feature_data_points = 0 # number of feature data analyses (some tracks do not have analyses available) # iterate until hit requested num of tracks for _ in range(0, tracks, limit): @@ -43,22 +42,27 @@ def parse_library(headers, tracks, library_stats, user): # TODO: refactor the for loop body into helper function # iterate through each track for track_dict in saved_tracks_response['items']: - num_samples += 1 # update artist info before track so that Track object can reference # Artist object track_artists = [] for artist_dict in track_dict['track']['artists']: - increase_artist_count(headers, artist_dict['name'], - artist_dict['id'], library_stats) - track_artists.append(Artist.objects.get_or_create( + artist_obj, artist_created = Artist.objects.get_or_create( artist_id=artist_dict['id'], name=artist_dict['name'], - )[0]) + ) + + update_artist_genre(headers, artist_obj) + # get_or_create() returns a tuple (obj, created) + track_artists.append(artist_obj) - track_obj = save_track_obj(track_dict['track'], track_artists, user) - get_track_info(track_dict['track'], library_stats, num_samples) - audio_features_dict = get_audio_features(headers, - track_dict['track']['id'], track_obj) + track_obj, track_created = save_track_obj(track_dict['track'], track_artists, user) + + # if a new track is not created, the associated audio feature does not need to be created again + if track_created: + save_audio_features(headers, track_dict['track']['id'], track_obj) + """ + TODO: Put this login in another function + # Audio analysis could be empty if not present in Spotify database if len(audio_features_dict) != 0: # Track the number of audio analyses for calculating # audio feature averages and standard deviations on the fly @@ -66,62 +70,55 @@ def parse_library(headers, tracks, library_stats, user): for feature, feature_data in audio_features_dict.items(): update_audio_feature_stats(feature, feature_data, feature_data_points, library_stats) - + """ # calculates num_songs with offset + songs retrieved - library_stats['num_songs'] = offset + len(saved_tracks_response['items']) offset += limit - calculate_genres_from_artists(headers, library_stats) + # calculate_genres_from_artists(headers, library_stats) # pprint.pprint(library_stats) # }}} parse_library # # save_track_obj {{{ # + def save_track_obj(track_dict, artists, user): """Make an entry in the database for this track if it doesn't exist already. :track_dict: dictionary from the API call containing track information. :artists: artists of the song, passed in as a list of Artist objects. :user: User object for which this Track is to be associated with. - :returns: The created/retrieved Track object. + :returns: (The created/retrieved Track object, created) """ - track_obj_query = Track.objects.filter(track_id__exact=track_dict['id']) - if len(track_obj_query) == 0: - new_track = Track.objects.create( - track_id=track_dict['id'], - year=track_dict['album']['release_date'].split('-')[0], - popularity=int(track_dict['popularity']), - runtime=int(float(track_dict['duration_ms']) / 1000), - name=track_dict['name'], - ) - # print("pop/run: ", new_track.popularity, new_track.runtime) - - # have to add artists and user after saving object since track needs to - # have ID before filling in m2m field + new_track, created = Track.objects.get_or_create( + track_id=track_dict['id'], + year=track_dict['album']['release_date'].split('-')[0], + popularity=int(track_dict['popularity']), + runtime=int(float(track_dict['duration_ms']) / 1000), + name=track_dict['name'], + ) + + # have to add artists and user after saving object since track needs to + # have ID before filling in m2m field + if created: for artist in artists: new_track.artists.add(artist) new_track.users.add(user) new_track.save() - return new_track - elif len(track_obj_query) == 1: - return track_obj_query[0] + return new_track, created # }}} save_track_obj # # get_audio_features {{{ # -def get_audio_features(headers, track_id, track): - """Returns the audio features of a soundtrack +def save_audio_features(headers, track_id, track): + """Creates and saves a new AudioFeatures object Args: headers: headers containing the API token track_id: the id of the soundtrack, needed to query the Spotify API - track: Track object to associate with the AudioFeatures object + track: Track object to associate with the new AudioFeatures object - Returns: - A dictionary with the features as its keys, if audio feature data is missing for the track, - an empty dictionary is returned. """ response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json() @@ -141,7 +138,6 @@ def get_audio_features(headers, track_id, track): setattr(audio_features_entry, key, val) audio_features_entry.save() - return features_dict # }}} get_audio_features # @@ -300,30 +296,27 @@ def get_track_info(track_dict, library_stats, sample_size): # }}} get_track_info # -# calculate_genres_from_artists {{{ # +# update_genres_from_artists {{{ # + -def calculate_genres_from_artists(headers, library_stats): - """Tallies up genre counts based on artists in library_stats. +def update_artist_genre(headers, artist_obj): + """Updates the top genre for an artist by querying the Spotify API :headers: For making the API call. - :library_stats: Dictionary containing the data mined from user's Spotify library + :artist_obj: the Artist object whose genre field will be updated :returns: None """ - for artist_entry in library_stats['artists'].values(): - artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_entry['id'], headers=headers).json() - # increase each genre count by artist count - for genre in artist_response['genres']: - increase_nested_key('genres', genre, library_stats, artist_entry['count']) - - # update genre for artist in database with top genre - Artist.objects.filter(artist_id=artist_entry['id']).update(genre=artist_response['genres'][0]) + artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_obj.id, headers=headers).json() + # update genre for artist in database with top genre + artist_obj.update(genre=artist_response['genres'][0]) # }}} calculate_genres_from_artists # # process_library_stats {{{ # + def process_library_stats(library_stats): """Processes library_stats into format more suitable for D3 consumption @@ -388,23 +381,3 @@ def get_genre_data(user): # user_tracks = Track.objects.filter(users__exact=user) # for track in user_tracks: # print(track.name) - - -def get_artist_data(user_id): - """Return artist data needed to create the graph for user. - - :user_id: user ID for which to return the data for. - :returns: List of dicts containing counts for each artist. - - """ - # TODO: not actual artists for user - # PICK UP: figure out how to pass data to D3/frontend - print(user_id) - # user = User.objects.get(user_id=user_id) - artist_counts = Artist.objects.annotate(num_songs=Count('track')) - processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] - # for artist in artist_counts: - # print(artist.name, artist.num_songs) - return JsonResponse(processed_artist_data, safe=False) - # return serializers.serialize('json', processed_artist_data) - # return processed_artist_data diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 3ab4f61..6bf090b 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -10,9 +10,9 @@ import pprint from datetime import datetime from django.shortcuts import render, redirect -from django.http import HttpResponse, HttpResponseBadRequest +from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.db.models import Count -from .utils import parse_library, process_library_stats, get_artist_data +from .utils import parse_library, process_library_stats from .models import User, Track, AudioFeatures, Artist # }}} imports # @@ -44,6 +44,7 @@ def generate_random_string(length): # token_expired {{{ # + def token_expired(token_obtained_at, valid_for): """Returns True if token expired, False if otherwise @@ -143,6 +144,8 @@ def user_data(request): user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json() request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model # request.session['user_name'] = user_data_response['display_name'] + + # get_or_create() returns a tuple (obj, created) user = User.objects.get_or_create(user_id=user_data_response['id'])[0] context = { @@ -174,8 +177,20 @@ def user_data(request): def test_db(request): user_id = "polarbier" context = { - 'artist_data': get_artist_data(user_id), 'user_id': user_id, } # get_artist_data(user) return render(request, 'spotifyvis/test_db.html', context) + + +def get_artist_data(request, user_id): + + # TODO: not actual artists for user + # PICK UP: figure out how to pass data to D3/frontend + print(user_id) + # user = User.objects.get(user_id=user_id) + artist_counts = Artist.objects.annotate(num_songs=Count('track')) + processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] + # for artist in artist_counts: + # print(artist.name, artist.num_songs) + return JsonResponse(data=processed_artist_data, safe=False) From 90dbda4336075a297cb6ceb437298077f5d59701 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 8 Jun 2018 19:47:51 -0400 Subject: [PATCH 10/49] Removed UserAudioSummary class --- spotifyvis/models.py | 40 +++------------------------------------- spotifyvis/utils.py | 2 +- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/spotifyvis/models.py b/spotifyvis/models.py index aacfd1e..f09c87a 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -4,6 +4,7 @@ from django.db import models MAX_ID = 30 # Artist {{{ # + class Artist(models.Model): class Meta: verbose_name = "Artist" @@ -58,6 +59,7 @@ class Track(models.Model): # AudioFeatures {{{ # + class AudioFeatures(models.Model): class Meta: @@ -77,40 +79,4 @@ class AudioFeatures(models.Model): def __str__(self): return super(AudioFeatures, self).__str__() -# }}} AudioFeatures # - - -''' -class UserAudioSummary(models.Model): - """ - Stores the summary of a user's audio preferences - """ - class Meta: - verbose_name = "AudioFeatures" - verbose_name_plural = "AudioFeatures" - - user = models.OneToOneField(User, on_delete=models.CASCADE, primary_key=True,) - avg_acousticness = models.DecimalField(decimal_places=3, max_digits=3) - stdev_acousticness = models.DecimalField(decimal_places=3, max_digits=3) - - avg_danceability = models.DecimalField(decimal_places=3, max_digits=3) - stdev_danceability = models.DecimalField(decimal_places=3, max_digits=3) - - avg_energy = models.DecimalField(decimal_places=3, max_digits=3) - stdev_energy = models.DecimalField(decimal_places=3, max_digits=3) - - avg_instrumentalness = models.DecimalField(decimal_places=3, max_digits=3) - stdev_instrumentalness = models.DecimalField(decimal_places=3, max_digits=3) - - avg_loudness = models.DecimalField(decimal_places=3, max_digits=6) - stdev_loudness = models.DecimalField(decimal_places=3, max_digits=6) - - avg_speechiness = models.DecimalField(decimal_places=3, max_digits=3) - stdev_speechiness = models.DecimalField(decimal_places=3, max_digits=3) - - avg_tempo = models.DecimalField(decimal_places=3, max_digits=6) - stdev_tempo = models.DecimalField(decimal_places=3, max_digits=6) - - avg_valence = models.DecimalField(decimal_places=3, max_digits=3) - stdev_valence = models.DecimalField(decimal_places=3, max_digits=3) -''' \ No newline at end of file +# }}} AudioFeatures # \ No newline at end of file diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 1d61a9e..2362c1d 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -61,7 +61,7 @@ def parse_library(headers, tracks, library_stats, user): if track_created: save_audio_features(headers, track_dict['track']['id'], track_obj) """ - TODO: Put this login in another function + TODO: Put this logic in another function # Audio analysis could be empty if not present in Spotify database if len(audio_features_dict) != 0: # Track the number of audio analyses for calculating From 0e69d3d1905ed69f2027514fb692f413f6121518 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 8 Jun 2018 19:50:50 -0400 Subject: [PATCH 11/49] Removed test_db.js --- spotifyvis/static/spotifyvis/scripts/test_db.js | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 spotifyvis/static/spotifyvis/scripts/test_db.js diff --git a/spotifyvis/static/spotifyvis/scripts/test_db.js b/spotifyvis/static/spotifyvis/scripts/test_db.js deleted file mode 100644 index cb5fb67..0000000 --- a/spotifyvis/static/spotifyvis/scripts/test_db.js +++ /dev/null @@ -1,5 +0,0 @@ -d3.json("{% url "get_artist_data", user_id %}").then(function(error, data) { - data.forEach(function(d) { - console.log(d.name, d.num_songs); - }); -}); From fd7a031bf2b3b0e9be24e516b8a4745cf088328a Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 8 Jun 2018 19:58:44 -0400 Subject: [PATCH 12/49] Fix incorrect usage of update() update() is a QuerySet method. To update a single object, use save(). --- spotifyvis/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 2362c1d..b23b3c0 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -310,7 +310,8 @@ def update_artist_genre(headers, artist_obj): """ artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_obj.id, headers=headers).json() # update genre for artist in database with top genre - artist_obj.update(genre=artist_response['genres'][0]) + artist_obj.genre = artist_response['genres'][0] + artist_obj.save() # }}} calculate_genres_from_artists # From 92d5d174e814ffbe6513875f29f7ebc63ed24bb5 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 8 Jun 2018 20:08:17 -0400 Subject: [PATCH 13/49] Modify maximum field length Artist.genre and Track.name had maximum lengths that were not long enough, now increased. --- .../migrations/0002_auto_20180608_2002.py | 18 ++++++++++++++++++ .../migrations/0003_auto_20180608_2007.py | 18 ++++++++++++++++++ spotifyvis/models.py | 4 ++-- spotifyvis/utils.py | 3 ++- 4 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 spotifyvis/migrations/0002_auto_20180608_2002.py create mode 100644 spotifyvis/migrations/0003_auto_20180608_2007.py diff --git a/spotifyvis/migrations/0002_auto_20180608_2002.py b/spotifyvis/migrations/0002_auto_20180608_2002.py new file mode 100644 index 0000000..b7e48e7 --- /dev/null +++ b/spotifyvis/migrations/0002_auto_20180608_2002.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.5 on 2018-06-09 00:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('spotifyvis', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='artist', + name='genre', + field=models.CharField(max_length=30), + ), + ] diff --git a/spotifyvis/migrations/0003_auto_20180608_2007.py b/spotifyvis/migrations/0003_auto_20180608_2007.py new file mode 100644 index 0000000..323ca26 --- /dev/null +++ b/spotifyvis/migrations/0003_auto_20180608_2007.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.5 on 2018-06-09 00:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('spotifyvis', '0002_auto_20180608_2002'), + ] + + operations = [ + migrations.AlterField( + model_name='track', + name='name', + field=models.CharField(max_length=150), + ), + ] diff --git a/spotifyvis/models.py b/spotifyvis/models.py index f09c87a..07f73ff 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -13,7 +13,7 @@ class Artist(models.Model): artist_id = models.CharField(primary_key=True, max_length=MAX_ID) # unique since only storing one genre per artist right now name = models.CharField(unique=True, max_length=50) - genre = models.CharField(max_length=20) + genre = models.CharField(max_length=30) def __str__(self): return self.name @@ -49,7 +49,7 @@ class Track(models.Model): year = models.PositiveSmallIntegerField() popularity = models.PositiveSmallIntegerField() runtime = models.PositiveSmallIntegerField() - name = models.CharField(max_length=75) + name = models.CharField(max_length=150) users = models.ManyToManyField(User, blank=True) def __str__(self): diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index b23b3c0..40d116d 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -90,6 +90,7 @@ def save_track_obj(track_dict, artists, user): :returns: (The created/retrieved Track object, created) """ + print(track_dict['name']) new_track, created = Track.objects.get_or_create( track_id=track_dict['id'], year=track_dict['album']['release_date'].split('-')[0], @@ -308,7 +309,7 @@ def update_artist_genre(headers, artist_obj): :returns: None """ - artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_obj.id, headers=headers).json() + artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_obj.artist_id, headers=headers).json() # update genre for artist in database with top genre artist_obj.genre = artist_response['genres'][0] artist_obj.save() From 53f78853d92eabc660770c7faf1d6c22be823db2 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 8 Jun 2018 20:59:27 -0400 Subject: [PATCH 14/49] Delete 0001_initial.py --- spotifyvis/migrations/0001_initial.py | 79 --------------------------- 1 file changed, 79 deletions(-) delete mode 100644 spotifyvis/migrations/0001_initial.py diff --git a/spotifyvis/migrations/0001_initial.py b/spotifyvis/migrations/0001_initial.py deleted file mode 100644 index fb92c44..0000000 --- a/spotifyvis/migrations/0001_initial.py +++ /dev/null @@ -1,79 +0,0 @@ -# Generated by Django 2.0.5 on 2018-06-07 00:52 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Artist', - fields=[ - ('artist_id', models.CharField(max_length=30, primary_key=True, serialize=False)), - ('name', models.CharField(max_length=50, unique=True)), - ('genre', models.CharField(max_length=20)), - ], - options={ - 'verbose_name': 'Artist', - 'verbose_name_plural': 'Artists', - }, - ), - migrations.CreateModel( - name='Track', - fields=[ - ('track_id', models.CharField(max_length=30, primary_key=True, serialize=False)), - ('year', models.PositiveSmallIntegerField()), - ('popularity', models.PositiveSmallIntegerField()), - ('runtime', models.PositiveSmallIntegerField()), - ('name', models.CharField(max_length=75)), - ], - options={ - 'verbose_name': 'Track', - 'verbose_name_plural': 'Tracks', - }, - ), - migrations.CreateModel( - name='User', - fields=[ - ('user_id', models.CharField(max_length=30, primary_key=True, serialize=False)), - ], - options={ - 'verbose_name': 'User', - 'verbose_name_plural': 'Users', - }, - ), - migrations.CreateModel( - name='AudioFeatures', - fields=[ - ('track', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='spotifyvis.Track')), - ('acousticness', models.DecimalField(decimal_places=3, max_digits=3)), - ('danceability', models.DecimalField(decimal_places=3, max_digits=3)), - ('energy', models.DecimalField(decimal_places=3, max_digits=3)), - ('instrumentalness', models.DecimalField(decimal_places=3, max_digits=3)), - ('loudness', models.DecimalField(decimal_places=3, max_digits=6)), - ('speechiness', models.DecimalField(decimal_places=3, max_digits=3)), - ('tempo', models.DecimalField(decimal_places=3, max_digits=6)), - ('valence', models.DecimalField(decimal_places=3, max_digits=3)), - ], - options={ - 'verbose_name': 'AudioFeatures', - 'verbose_name_plural': 'AudioFeatures', - }, - ), - migrations.AddField( - model_name='track', - name='artists', - field=models.ManyToManyField(blank=True, to='spotifyvis.Artist'), - ), - migrations.AddField( - model_name='track', - name='users', - field=models.ManyToManyField(blank=True, to='spotifyvis.User'), - ), - ] From d995a1544442854ce8004b44d66c9a867912e1f4 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 8 Jun 2018 20:59:34 -0400 Subject: [PATCH 15/49] Delete 0002_auto_20180608_2002.py --- .../migrations/0002_auto_20180608_2002.py | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 spotifyvis/migrations/0002_auto_20180608_2002.py diff --git a/spotifyvis/migrations/0002_auto_20180608_2002.py b/spotifyvis/migrations/0002_auto_20180608_2002.py deleted file mode 100644 index b7e48e7..0000000 --- a/spotifyvis/migrations/0002_auto_20180608_2002.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.0.5 on 2018-06-09 00:02 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('spotifyvis', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='artist', - name='genre', - field=models.CharField(max_length=30), - ), - ] From ef7933fb6742324882dd5d1b2509f4d8811e982f Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 8 Jun 2018 20:59:42 -0400 Subject: [PATCH 16/49] Delete 0003_auto_20180608_2007.py --- .../migrations/0003_auto_20180608_2007.py | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 spotifyvis/migrations/0003_auto_20180608_2007.py diff --git a/spotifyvis/migrations/0003_auto_20180608_2007.py b/spotifyvis/migrations/0003_auto_20180608_2007.py deleted file mode 100644 index 323ca26..0000000 --- a/spotifyvis/migrations/0003_auto_20180608_2007.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.0.5 on 2018-06-09 00:07 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('spotifyvis', '0002_auto_20180608_2002'), - ] - - operations = [ - migrations.AlterField( - model_name='track', - name='name', - field=models.CharField(max_length=150), - ), - ] From ea5990d048ae08d47ebe976c73bb02654fd80ac4 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Fri, 8 Jun 2018 22:30:01 -0400 Subject: [PATCH 17/49] Fixed duplicate key error in save_track_obj Can't create Track object without the artists/user so get_or_create doesn't work properly. --- spotifyvis/models.py | 3 ++- spotifyvis/utils.py | 28 +++++++++++++++------------- spotifyvis/views.py | 8 ++++---- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 07f73ff..75ef2b7 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -2,6 +2,7 @@ from django.db import models # id's are 22 in length in examples but set to 30 for buffer MAX_ID = 30 + # Artist {{{ # @@ -79,4 +80,4 @@ class AudioFeatures(models.Model): def __str__(self): return super(AudioFeatures, self).__str__() -# }}} AudioFeatures # \ No newline at end of file +# }}} AudioFeatures # diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 40d116d..11c75d8 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -80,7 +80,6 @@ def parse_library(headers, tracks, library_stats, user): # save_track_obj {{{ # - def save_track_obj(track_dict, artists, user): """Make an entry in the database for this track if it doesn't exist already. @@ -91,22 +90,25 @@ def save_track_obj(track_dict, artists, user): """ print(track_dict['name']) - new_track, created = Track.objects.get_or_create( - track_id=track_dict['id'], - year=track_dict['album']['release_date'].split('-')[0], - popularity=int(track_dict['popularity']), - runtime=int(float(track_dict['duration_ms']) / 1000), - name=track_dict['name'], - ) - - # have to add artists and user after saving object since track needs to - # have ID before filling in m2m field - if created: + track_query = Track.objects.filter(track_id__exact=track_dict['id']) + if len(track_query) != 0: + return track_query[0], False + else: + new_track = Track.objects.create( + track_id=track_dict['id'], + year=track_dict['album']['release_date'].split('-')[0], + popularity=int(track_dict['popularity']), + runtime=int(float(track_dict['duration_ms']) / 1000), + name=track_dict['name'], + ) + + # have to add artists and user after saving object since track needs to + # have ID before filling in m2m field for artist in artists: new_track.artists.add(artist) new_track.users.add(user) new_track.save() - return new_track, created + return new_track, True # }}} save_track_obj # diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 6bf090b..5561feb 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -175,6 +175,8 @@ def user_data(request): # }}} user_data # def test_db(request): + """TODO + """ user_id = "polarbier" context = { 'user_id': user_id, @@ -184,13 +186,11 @@ def test_db(request): def get_artist_data(request, user_id): - + """TODO + """ # TODO: not actual artists for user - # PICK UP: figure out how to pass data to D3/frontend print(user_id) # user = User.objects.get(user_id=user_id) artist_counts = Artist.objects.annotate(num_songs=Count('track')) processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] - # for artist in artist_counts: - # print(artist.name, artist.num_songs) return JsonResponse(data=processed_artist_data, safe=False) From 9d430480678fb2face146f859986bc4b4056817d Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Sat, 9 Jun 2018 13:58:52 -0400 Subject: [PATCH 18/49] Clean up views.py and utils.py further views.py and utils.py were cleaned up further to remove any obsolete usage of library_stats. --- spotifyvis/templates/spotifyvis/test_db.html | 9 -------- spotifyvis/utils.py | 9 ++------ spotifyvis/views.py | 22 +++----------------- 3 files changed, 5 insertions(+), 35 deletions(-) diff --git a/spotifyvis/templates/spotifyvis/test_db.html b/spotifyvis/templates/spotifyvis/test_db.html index 0c51c81..ba3dcaa 100644 --- a/spotifyvis/templates/spotifyvis/test_db.html +++ b/spotifyvis/templates/spotifyvis/test_db.html @@ -11,15 +11,6 @@ - -
      - {% for artist in artist_data %} -
    • {{ artist.name }} - {{ artist.num_songs }}
      • - {% endfor %} -
    - 
     {% filter force_escape %} {% debug %} {% endfilter %}
    + diff --git a/spotifyvis/urls.py b/spotifyvis/urls.py index 55380bd..eb7e299 100644 --- a/spotifyvis/urls.py +++ b/spotifyvis/urls.py @@ -10,4 +10,6 @@ urlpatterns = [ path('user_data', user_data, name='user_data'), path('test_db', test_db, name='test_db'), path('user_artists/', get_artist_data, name='get_artist_data'), + path('audio_features//', get_audio_feature_data, name='get_audio_feature_data'), + ] diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 30da1b6..1385371 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -105,7 +105,7 @@ def callback(request): 'client_secret': os.environ['SPOTIFY_CLIENT_SECRET'], } - response = requests.post('https://accounts.spotify.com/api/token', data = payload).json() + response = requests.post('https://accounts.spotify.com/api/token', data=payload).json() # despite its name, datetime.today() returns a datetime object, not a date object # use datetime.strptime() to get a datetime object from a string request.session['token_obtained_at'] = datetime.strftime(datetime.today(), TIME_FORMAT) @@ -142,15 +142,18 @@ def user_data(request): } user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json() - request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model + request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model # request.session['user_name'] = user_data_response['display_name'] - # get_or_create() returns a tuple (obj, created) - user = User.objects.get_or_create(user_id=user_data_response['id'])[0] + try: + user = User.objects.get(user_id=user_data_response['id']) + except User.DoesNotExist: + user = User(user_id=user_data_response['id'], user_secret=generate_random_string(30)) + user.save() context = { - 'user_name': user_data_response['display_name'], - 'id': user_data_response['id'], + 'id': user_data_response['id'], + 'user_secret': user.user_secret, } parse_library(headers, TRACKS_TO_QUERY, user) @@ -177,4 +180,24 @@ def get_artist_data(request, user_id): artist_counts = Artist.objects.annotate(num_songs=Count('track')) processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] - return JsonResponse(data=processed_artist_data, safe=False) + return JsonResponse(data=processed_artist_data, safe=False) + + +def get_audio_feature_data(request, audio_feature, client_secret): + """Returns all data points for a given audio feature + + Args: + request: the HTTP request + audio_feature: The audio feature to be queried + client_secret: client secret, used to identify the user + """ + user = User.objects.get(user_secret=client_secret) + user_tracks = Track.objects.filter(users=user) + response_payload = { + 'data_points': [], + } + for track in user_tracks: + audio_feature_obj = AudioFeatures.objects.get(track=track) + response_payload['data_points'].append(getattr(audio_feature_obj, audio_feature)) + return JsonResponse(response_payload) + From 4698663a85a6327fe8146df26f58a5d7f6b43b59 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Sun, 10 Jun 2018 09:21:35 -0400 Subject: [PATCH 20/49] Rewrite generate_random_string() Rewrote generate_random_string() in a more Pythonic fashion. --- spotifyvis/views.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 1385371..12774d4 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -7,6 +7,7 @@ import os import urllib import json import pprint +import string from datetime import datetime from django.shortcuts import render, redirect @@ -32,11 +33,8 @@ def generate_random_string(length): Returns: A random string """ - rand_str = "" - possible_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - - for _ in range(length): - rand_str += possible_chars[random.randint(0, len(possible_chars) - 1)] + all_chars = string.ascii_letters + string.digits + string.punctuation + rand_str = "".join(random.choice(all_chars) for _ in range(length)) return rand_str From 05b5cc404a3e2fa6db840a37e496008627cef7e4 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Mon, 11 Jun 2018 21:58:51 -0400 Subject: [PATCH 21/49] Trying to get correct artist breakdown in genres Counts are off since there are multiple artists on a track. --- spotifyvis/templates/spotifyvis/test_db.html | 49 +++++++++----------- spotifyvis/utils.py | 20 +++++++- spotifyvis/views.py | 16 ++++--- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/spotifyvis/templates/spotifyvis/test_db.html b/spotifyvis/templates/spotifyvis/test_db.html index 0bf5b7f..0f481bd 100644 --- a/spotifyvis/templates/spotifyvis/test_db.html +++ b/spotifyvis/templates/spotifyvis/test_db.html @@ -1,31 +1,26 @@ - + - - - - - Test DB Page - - - - - - - + + + + + + Test DB Page + + + + + + + + diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 2caf39d..8572444 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -4,7 +4,7 @@ import math import pprint from .models import Artist, User, Track, AudioFeatures -from django.db.models import Count +from django.db.models import Count, Q from django.http import JsonResponse from django.core import serializers import json @@ -317,7 +317,7 @@ def update_artist_genre(headers, artist_obj): # }}} # -# {{{ # +# get_top_genre {{{ # def get_top_genre(headers, top_artist_id): """Updates the top genre for a track by querying the Spotify API @@ -393,3 +393,19 @@ def process_library_stats(library_stats): # }}} process_library_stats # +def get_artists_in_genre(user, genre): + """Return count of artists in genre. + + :genre: genre to count artists for. + + :returns: dict of artists in the genre along with the number of songs they + have. + """ + artist_counts = (Artist.objects.filter(track__users=user) + .filter(track__genre=genre) + # .annotate(num_songs=Count('track', filter=Q(track__genre=genre))) + .annotate(num_songs=Count('track')) + ) + processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] + # pprint.pprint(processed_artist_counts) + return processed_artist_counts diff --git a/spotifyvis/views.py b/spotifyvis/views.py index c2a2ae2..c14b5ad 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -13,7 +13,7 @@ from datetime import datetime from django.shortcuts import render, redirect from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.db.models import Count, Q -from .utils import parse_library, process_library_stats +from .utils import parse_library, process_library_stats, get_artists_in_genre from .models import User, Track, AudioFeatures, Artist # }}} imports # @@ -164,7 +164,8 @@ def user_data(request): def test_db(request): """TODO """ - user_id = "polarbier" + # user_id = "polarbier" + user_id = "35kxo00qqo9pd1comj6ylxjq7" context = { 'user_secret': User.objects.get(user_id=user_id).user_secret, } @@ -180,8 +181,9 @@ def get_artist_data(request, user_secret): user = User.objects.get(user_id=user_secret) artist_counts = Artist.objects.annotate(num_songs=Count('track', filter=Q(track__users=user))) - processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] - return JsonResponse(data=processed_artist_data, safe=False) + processed_artist_counts = [{'name': artist.name, + 'num_songs': artist.num_songs} for artist in artist_counts] + return JsonResponse(data=processed_artist_counts, safe=False) # }}} get_artist_data # @@ -214,12 +216,14 @@ def get_genre_data(request, user_secret): TODO """ user = User.objects.get(user_secret=user_secret) - genre_counts = (Track.objects.filter(users=user) + genre_counts = (Track.objects.filter(users__exact=user) .values('genre') .order_by('genre') .annotate(num_songs=Count('genre')) ) - # pprint.pprint(genre_counts) + for genre_dict in genre_counts: + genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre']) + pprint.pprint(list(genre_counts)) return JsonResponse(data=list(genre_counts), safe=False) # }}} get_genre_data # From 7b968c9d868e08fbaac6a489f1e6ec5f6e452dc9 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Mon, 11 Jun 2018 21:58:51 -0400 Subject: [PATCH 22/49] Include artist breakdown in get_genre_data --- spotifyvis/templates/spotifyvis/test_db.html | 49 +++++++++----------- spotifyvis/utils.py | 24 ++++++++-- spotifyvis/views.py | 14 ++++-- 3 files changed, 52 insertions(+), 35 deletions(-) diff --git a/spotifyvis/templates/spotifyvis/test_db.html b/spotifyvis/templates/spotifyvis/test_db.html index 0bf5b7f..0f481bd 100644 --- a/spotifyvis/templates/spotifyvis/test_db.html +++ b/spotifyvis/templates/spotifyvis/test_db.html @@ -1,31 +1,26 @@ - + - - - - - Test DB Page - - - - - - - + + + + + + Test DB Page + + + + + + + + diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 2caf39d..f58f549 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -4,7 +4,7 @@ import math import pprint from .models import Artist, User, Track, AudioFeatures -from django.db.models import Count +from django.db.models import Count, Q from django.http import JsonResponse from django.core import serializers import json @@ -317,7 +317,7 @@ def update_artist_genre(headers, artist_obj): # }}} # -# {{{ # +# get_top_genre {{{ # def get_top_genre(headers, top_artist_id): """Updates the top genre for a track by querying the Spotify API @@ -339,7 +339,6 @@ def get_top_genre(headers, top_artist_id): # process_library_stats {{{ # - def process_library_stats(library_stats): """Processes library_stats into format more suitable for D3 consumption @@ -393,3 +392,22 @@ def process_library_stats(library_stats): # }}} process_library_stats # +# get_artists_in_genre {{{ # + +def get_artists_in_genre(user, genre): + """Return count of artists in genre. + + :genre: genre to count artists for. + + :returns: dict of artists in the genre along with the number of songs they + have. + """ + artist_counts = (Artist.objects.filter(track__users=user) + .filter(track__genre=genre) + .annotate(num_songs=Count('track', distinct=True)) + ) + processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] + # pprint.pprint(processed_artist_counts) + return processed_artist_counts + +# }}} get_artists_in_genre # diff --git a/spotifyvis/views.py b/spotifyvis/views.py index c2a2ae2..ebe28c8 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -13,7 +13,7 @@ from datetime import datetime from django.shortcuts import render, redirect from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.db.models import Count, Q -from .utils import parse_library, process_library_stats +from .utils import parse_library, process_library_stats, get_artists_in_genre from .models import User, Track, AudioFeatures, Artist # }}} imports # @@ -165,6 +165,7 @@ def test_db(request): """TODO """ user_id = "polarbier" + # user_id = "35kxo00qqo9pd1comj6ylxjq7" context = { 'user_secret': User.objects.get(user_id=user_id).user_secret, } @@ -180,8 +181,9 @@ def get_artist_data(request, user_secret): user = User.objects.get(user_id=user_secret) artist_counts = Artist.objects.annotate(num_songs=Count('track', filter=Q(track__users=user))) - processed_artist_data = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] - return JsonResponse(data=processed_artist_data, safe=False) + processed_artist_counts = [{'name': artist.name, + 'num_songs': artist.num_songs} for artist in artist_counts] + return JsonResponse(data=processed_artist_counts, safe=False) # }}} get_artist_data # @@ -214,12 +216,14 @@ def get_genre_data(request, user_secret): TODO """ user = User.objects.get(user_secret=user_secret) - genre_counts = (Track.objects.filter(users=user) + genre_counts = (Track.objects.filter(users__exact=user) .values('genre') .order_by('genre') .annotate(num_songs=Count('genre')) ) - # pprint.pprint(genre_counts) + for genre_dict in genre_counts: + genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre']) + pprint.pprint(list(genre_counts)) return JsonResponse(data=list(genre_counts), safe=False) # }}} get_genre_data # From 2342e6ef49e4b4d018261d9ee7090dde8d43bd80 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 14 Jun 2018 05:31:07 -0400 Subject: [PATCH 23/49] Initial setup of stacked bar chart for genres Bunch of issues with it that still need to be fixed. --- sample-track-obj.json => sample-track-obj.py | 0 spotifyvis/templates/spotifyvis/test_db.html | 83 +++++++++++++++++++- spotifyvis/utils.py | 3 +- 3 files changed, 84 insertions(+), 2 deletions(-) rename sample-track-obj.json => sample-track-obj.py (100%) diff --git a/sample-track-obj.json b/sample-track-obj.py similarity index 100% rename from sample-track-obj.json rename to sample-track-obj.py diff --git a/spotifyvis/templates/spotifyvis/test_db.html b/spotifyvis/templates/spotifyvis/test_db.html index 0f481bd..59636c9 100644 --- a/spotifyvis/templates/spotifyvis/test_db.html +++ b/spotifyvis/templates/spotifyvis/test_db.html @@ -1,3 +1,5 @@ + + @@ -11,15 +13,94 @@ + + diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index f58f549..e823eaa 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -406,7 +406,8 @@ def get_artists_in_genre(user, genre): .filter(track__genre=genre) .annotate(num_songs=Count('track', distinct=True)) ) - processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] + # processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] + processed_artist_counts = {artist.name: artist.num_songs for artist in artist_counts} # pprint.pprint(processed_artist_counts) return processed_artist_counts From 162b2dad37b974b2a1700481ff20062db12579f9 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 14 Jun 2018 06:13:42 -0400 Subject: [PATCH 24/49] Setup color range for bars Included a script to generate attractive colors. --- spotifyvis/templates/spotifyvis/test_db.html | 32 ++++++++++++-------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/spotifyvis/templates/spotifyvis/test_db.html b/spotifyvis/templates/spotifyvis/test_db.html index 59636c9..c0908da 100644 --- a/spotifyvis/templates/spotifyvis/test_db.html +++ b/spotifyvis/templates/spotifyvis/test_db.html @@ -15,27 +15,31 @@ + + - - + + diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index ede11c1..b7a95c7 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -26,7 +26,7 @@ def parse_library(headers, tracks, user): """ # TODO: implement importing entire library with 0 as tracks param # number of tracks to get with each call - limit = 5 + limit = 50 # keeps track of point to get songs from offset = 0 payload = {'limit': str(limit)} diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 878ace0..8b9e3b8 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -19,7 +19,7 @@ from .models import User, Track, AudioFeatures, Artist # }}} imports # TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' -TRACKS_TO_QUERY = 15 +TRACKS_TO_QUERY = 100 # generate_random_string {{{ # From c5185561263330a9154140c46b0a366f951e4ba5 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 15 Jun 2018 17:03:49 -0400 Subject: [PATCH 27/49] Draw audio feature bar charts Started work on drawing the bar charts for audio features. --- spotifyvis/admin.py | 5 ++ spotifyvis/models.py | 8 ++- .../static/spotifyvis/scripts/user_data.js | 8 +-- .../templates/spotifyvis/user_data.html | 63 ++++++++++++++++++- spotifyvis/utils.py | 9 ++- spotifyvis/views.py | 2 +- 6 files changed, 81 insertions(+), 14 deletions(-) diff --git a/spotifyvis/admin.py b/spotifyvis/admin.py index 8c38f3f..bd71265 100644 --- a/spotifyvis/admin.py +++ b/spotifyvis/admin.py @@ -1,3 +1,8 @@ from django.contrib import admin +from .models import Track, Artist, AudioFeatures, User # Register your models here. +admin.site.register(Track) +admin.site.register(Artist) +admin.site.register(AudioFeatures) +admin.site.register(User) diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 0cc8879..6e28a07 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -51,10 +51,14 @@ class Track(models.Model): runtime = models.PositiveSmallIntegerField() name = models.CharField(max_length=150) users = models.ManyToManyField(User, blank=True) - genre = models.CharField(max_length=30) + genre = models.CharField(max_length=30, default="") def __str__(self): - return self.name + track_str = "{}, genre: {}, artists: [".format(self.name, self.genre) + for artist in self.artists.all(): + track_str += "{}, ".format(artist.name) + track_str += "]" + return track_str # }}} Track # diff --git a/spotifyvis/static/spotifyvis/scripts/user_data.js b/spotifyvis/static/spotifyvis/scripts/user_data.js index 993fe04..84c5141 100644 --- a/spotifyvis/static/spotifyvis/scripts/user_data.js +++ b/spotifyvis/static/spotifyvis/scripts/user_data.js @@ -2,8 +2,9 @@ * Retrieves data for a specific audio feature for a certain user * @param audioFeature: the audio feature for which data will be retrieved * @param clientSecret: the client secret, needed for security + * @param chartElement: the SVG element in which the data will be plotted */ -function getAudioFeatureData(audioFeature, userSecret) { +function plotAudioFeatureData(audioFeature, userSecret, chartElement) { let httpRequest = new XMLHttpRequest(); /* * Handler for the response @@ -12,10 +13,7 @@ function getAudioFeatureData(audioFeature, userSecret) { if (httpRequest.readyState === XMLHttpRequest.DONE) { if (httpRequest.status === 200) { let responseData = JSON.parse(httpRequest.responseText); - // TODO: The data points need to be plotted instead - for (let data of responseData.data_points) { - console.log(data); - } + } else { alert("There was a problem with the login request, please try again!"); } diff --git a/spotifyvis/templates/spotifyvis/user_data.html b/spotifyvis/templates/spotifyvis/user_data.html index ce4799d..8dcfa70 100644 --- a/spotifyvis/templates/spotifyvis/user_data.html +++ b/spotifyvis/templates/spotifyvis/user_data.html @@ -16,10 +16,71 @@

    You are using an outdated browser. Please upgrade your browser to improve your experience.

    Logged in as {{ id }}

    + diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 8572444..94010c1 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -401,11 +401,10 @@ def get_artists_in_genre(user, genre): :returns: dict of artists in the genre along with the number of songs they have. """ - artist_counts = (Artist.objects.filter(track__users=user) - .filter(track__genre=genre) - # .annotate(num_songs=Count('track', filter=Q(track__genre=genre))) - .annotate(num_songs=Count('track')) - ) + artist_counts = (Artist.objects.filter(track__users=user).distinct() + .filter(track__genre=genre).distinct() + .annotate(num_songs=Count('track')) + ) processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] # pprint.pprint(processed_artist_counts) return processed_artist_counts diff --git a/spotifyvis/views.py b/spotifyvis/views.py index c14b5ad..e44d998 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -165,7 +165,7 @@ def test_db(request): """TODO """ # user_id = "polarbier" - user_id = "35kxo00qqo9pd1comj6ylxjq7" + user_id = "chrisshyi13" context = { 'user_secret': User.objects.get(user_id=user_id).user_secret, } From bb9709539844b852d11c2e6502ef2ac1dd37cd9d Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 15 Jun 2018 20:11:15 -0400 Subject: [PATCH 28/49] Draw bar chart for instrumentalness Instrumentalness bar chart for 15 songs has been drawn. The chart needs to be beautified with colors and the axis labels could use some styling. --- .../templates/spotifyvis/user_data.html | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/spotifyvis/templates/spotifyvis/user_data.html b/spotifyvis/templates/spotifyvis/user_data.html index 8dcfa70..c237c58 100644 --- a/spotifyvis/templates/spotifyvis/user_data.html +++ b/spotifyvis/templates/spotifyvis/user_data.html @@ -18,9 +18,10 @@

    Logged in as {{ id }}

    - From 840152b99e468f692488f627c151df10bda7c9aa Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Fri, 15 Jun 2018 20:48:35 -0400 Subject: [PATCH 30/49] Add title to audio feature graphs Audio feature bar graphs now have titles. --- .../templates/spotifyvis/user_data.html | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/spotifyvis/templates/spotifyvis/user_data.html b/spotifyvis/templates/spotifyvis/user_data.html index 34229c2..2bfc6da 100644 --- a/spotifyvis/templates/spotifyvis/user_data.html +++ b/spotifyvis/templates/spotifyvis/user_data.html @@ -20,6 +20,13 @@ - diff --git a/spotifyvis/templates/spotifyvis/logged_in.html b/spotifyvis/templates/spotifyvis/logged_in.html new file mode 100644 index 0000000..a3d9017 --- /dev/null +++ b/spotifyvis/templates/spotifyvis/logged_in.html @@ -0,0 +1,12 @@ + +{% load static %} + + + + Logged In + + + + Audio Features + + \ No newline at end of file diff --git a/spotifyvis/urls.py b/spotifyvis/urls.py index 4612571..f5bc82b 100644 --- a/spotifyvis/urls.py +++ b/spotifyvis/urls.py @@ -11,6 +11,7 @@ urlpatterns = [ path('test_db', test_db, name='test_db'), path('user_artists/', get_artist_data, name='get_artist_data'), path('user_genres/', get_genre_data, name='get_genre_data'), + path('audio_features/', audio_features, name='audio_features'), path('audio_features//', get_audio_feature_data, name='get_audio_feature_data'), ] diff --git a/spotifyvis/views.py b/spotifyvis/views.py index b5992b6..2fcc85e 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -65,6 +65,7 @@ def index(request): # login {{{ # + def login(request): # use a randomly generated state string to prevent cross-site request forgery attacks @@ -118,6 +119,7 @@ def callback(request): # user_data {{{ # + def user_data(request): token_obtained_at = datetime.strptime(request.session['token_obtained_at'], TIME_FORMAT) valid_for = int(request.session['valid_for']) @@ -130,7 +132,7 @@ def user_data(request): 'client_secret': os.environ['SPOTIFY_CLIENT_SECRET'] } - refresh_token_response = requests.post('https://accounts.spotify.com/api/token', data = req_body).json() + refresh_token_response = requests.post('https://accounts.spotify.com/api/token', data=req_body).json() request.session['access_token'] = refresh_token_response['access_token'] request.session['valid_for'] = refresh_token_response['expires_in'] @@ -139,7 +141,7 @@ def user_data(request): 'Authorization': auth_token_str } - user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json() + user_data_response = requests.get('https://api.spotify.com/v1/me', headers=headers).json() request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model # request.session['user_name'] = user_data_response['display_name'] @@ -148,15 +150,15 @@ def user_data(request): except User.DoesNotExist: # Python docs recommends 32 bytes of randomness against brute force attacks user = User(user_id=user_data_response['id'], user_secret=secrets.token_urlsafe(32)) + request.session['user_secret'] = user.user_secret user.save() context = { - 'id': user_data_response['id'], 'user_secret': user.user_secret, } parse_library(headers, TRACKS_TO_QUERY, user) - return render(request, 'spotifyvis/user_data.html', context) + return render(request, 'spotifyvis/logged_in.html', context) # }}} user_data # @@ -188,6 +190,16 @@ def get_artist_data(request, user_secret): # }}} get_artist_data # + +def audio_features(request, client_secret): + user = User.objects.get(user_secret=client_secret) + context = { + 'user_id': user.user_id, + 'user_secret': client_secret, + } + return render(request, "spotifyvis/audio_features.html", context) + + # get_audio_feature_data {{{ # def get_audio_feature_data(request, audio_feature, client_secret): From 4c55744db87b593a6a6a325f5a7b291d2c21cf71 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Mon, 25 Jun 2018 22:09:05 -0400 Subject: [PATCH 44/49] Update docstring for parse_library() The docstring of parse_library still mentions the library_stats dictionary, which had already been removed. --- spotifyvis/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 3bb47e3..f94f76f 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -15,7 +15,7 @@ import json def parse_library(headers, tracks, user): - """Scans user's library for certain number of tracks to update library_stats with. + """Scans user's library for certain number of tracks and store the information in a database :headers: For API call. :tracks: Number of tracks to get from user's library. From 3fe52343aa72a2dbdfca09db19c0cb88a677501e Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 27 Jun 2018 00:34:38 -0400 Subject: [PATCH 45/49] Use "several" API call to get audio features Closes #20. Also close #38 since parse_library is a bit cleaner now with some folds, and there's not much to be refactored any more. --- spotifyvis/utils.py | 101 +++++++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 34 deletions(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index a97a916..e8244b6 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -14,7 +14,8 @@ import json USER_TRACKS_LIMIT = 50 # ARTIST_LIMIT = 50 ARTIST_LIMIT = 25 -FEATURES_LIMIT = 100 +# FEATURES_LIMIT = 100 +FEATURES_LIMIT = 25 # parse_library {{{ # @@ -33,61 +34,66 @@ def parse_library(headers, tracks, user): offset = 0 payload = {'limit': str(USER_TRACKS_LIMIT)} artist_genre_queue = [] + features_queue = [] # iterate until hit requested num of tracks for _ in range(0, tracks, USER_TRACKS_LIMIT): payload['offset'] = str(offset) - # get current set of tracks - saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', headers=headers, params=payload).json() + saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', + headers=headers, + params=payload).json() - # TODO: refactor the for loop body into helper function - # iterate through each track for track_dict in saved_tracks_response['items']: + # add artists {{{ # + # update artist info before track so that Track object can reference # Artist object track_artists = [] for artist_dict in track_dict['track']['artists']: artist_obj, artist_created = Artist.objects.get_or_create( - artist_id=artist_dict['id'], - name=artist_dict['name'], - ) + artist_id=artist_dict['id'], + name=artist_dict['name'],) + # only add/tally up artist genres if new if artist_created: artist_genre_queue.append(artist_obj) if len(artist_genre_queue) == ARTIST_LIMIT: add_artist_genres(headers, artist_genre_queue) artist_genre_queue = [] - - # update_artist_genre(headers, artist_obj) - # get_or_create() returns a tuple (obj, created) track_artists.append(artist_obj) - # top_genre = get_top_genre(headers, - # track_dict['track']['artists'][0]['id']) + # }}} add artists # + + # WIP: get most common genre top_genre = "" track_obj, track_created = save_track_obj(track_dict['track'], track_artists, top_genre, user) - # if a new track is not created, the associated audio feature does not need to be created again - # if track_created: - save_audio_features(headers, track_dict['track']['id'], track_obj) - """ - TODO: Put this logic in another function - # Audio analysis could be empty if not present in Spotify database - if len(audio_features_dict) != 0: - # Track the number of audio analyses for calculating - # audio feature averages and standard deviations on the fly - feature_data_points += 1 - for feature, feature_data in audio_features_dict.items(): - update_audio_feature_stats(feature, feature_data, - feature_data_points, library_stats) - """ + # add audio features {{{ # + + # if a new track is not created, the associated audio feature does + # not need to be created again + if track_created: + features_queue.append(track_obj) + if len(features_queue) == FEATURES_LIMIT: + get_audio_features(headers, features_queue) + features_queue = [] + + # }}} add audio features # + # calculates num_songs with offset + songs retrieved offset += USER_TRACKS_LIMIT - # pprint.pprint(library_stats) - # update artists left in queue since there will be probably be leftover - # artists that didn't hit ARTIST_LIMIT - add_artist_genres(headers, artist_genre_queue) + # clean-up {{{ # + + # update remaining artists without genres and songs without features if + # there are any + if len(artist_genre_queue) > 0: + add_artist_genres(headers, artist_genre_queue) + if len(features_queue) > 0: + get_audio_features(headers, features_queue) + + # }}} clean-up # + update_track_genres(user) # }}} parse_library # @@ -118,6 +124,7 @@ def save_track_obj(track_dict, artists, top_genre, user): :artists: artists of the song, passed in as a list of Artist objects. :top_genre: top genre associated with this track (see get_top_genre). :user: User object for which this Track is to be associated with. + :returns: (The created/retrieved Track object, created) """ @@ -144,6 +151,33 @@ def save_track_obj(track_dict, artists, top_genre, user): # }}} save_track_obj # +def get_audio_features(headers, track_objs): + """Creates and saves a new AudioFeatures objects for the respective + track_objs. track_objs should contain the API limit for a single call + (FEATURES_LIMIT) for maximum efficiency. + + :headers: headers containing the API token + :track_objs: Track objects to associate with the new AudioFeatures object + + :returns: None + """ + track_ids = str.join(",", [track_obj.track_id for track_obj in track_objs]) + params = {'ids': track_ids} + features_response = requests.get("https://api.spotify.com/v1/audio-features", + headers=headers,params=params).json()['audio_features'] + # pprint.pprint(features_response) + + useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", ] + for i in range(len(track_objs)): + if features_response[i] is not None: + # Data that we don't need + cur_features_obj = AudioFeatures() + cur_features_obj.track = track_objs[i] + for key, val in features_response[i].items(): + if key not in useless_keys: + setattr(cur_features_obj, key, val) + cur_features_obj.save() + # get_audio_features {{{ # def save_audio_features(headers, track_id, track): @@ -157,8 +191,6 @@ def save_audio_features(headers, track_id, track): """ response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json() - if track_id is '5S1IUPueD0xE0vj4zU3nSf': - pprint.pprint(response) if 'error' in response: return @@ -383,10 +415,11 @@ def add_artist_genres(headers, artist_objs): """ artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs]) - # print(len(artist_objs), artist_ids) + print(len(artist_objs), artist_ids) params = {'ids': artist_ids} artists_response = requests.get('https://api.spotify.com/v1/artists/', headers=headers, params=params).json()['artists'] + # pprint.pprint(artists_response) for i in range(len(artist_objs)): for genre in artists_response[i]['genres']: genre_obj, created = Genre.objects.get_or_create(name=genre, From 85c178f2b3ce89fc299d87f9ad96c75c44fdd8b2 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 27 Jun 2018 00:49:29 -0400 Subject: [PATCH 46/49] Resolves #39 --- spotifyvis/utils.py | 269 ++------------------------------------------ spotifyvis/views.py | 16 ++- 2 files changed, 21 insertions(+), 264 deletions(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index e8244b6..f8e2599 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -98,6 +98,8 @@ def parse_library(headers, tracks, user): # }}} parse_library # +# update_track_genres {{{ # + def update_track_genres(user): """Updates user's tracks with the most common genre associated with the songs' artist(s). @@ -115,6 +117,8 @@ def update_track_genres(user): track.save() # print(track_artists, track.genre) +# }}} update_track_genres # + # save_track_obj {{{ # def save_track_obj(track_dict, artists, top_genre, user): @@ -151,6 +155,8 @@ def save_track_obj(track_dict, artists, top_genre, user): # }}} save_track_obj # +# get_audio_features {{{ # + def get_audio_features(headers, track_objs): """Creates and saves a new AudioFeatures objects for the respective track_objs. track_objs should contain the API limit for a single call @@ -178,212 +184,9 @@ def get_audio_features(headers, track_objs): setattr(cur_features_obj, key, val) cur_features_obj.save() -# get_audio_features {{{ # - -def save_audio_features(headers, track_id, track): - """Creates and saves a new AudioFeatures object - - Args: - headers: headers containing the API token - track_id: the id of the soundtrack, needed to query the Spotify API - track: Track object to associate with the new AudioFeatures object - - """ - - response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json() - if 'error' in response: - return - - # Data that we don't need - useless_keys = [ - "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", - ] - audio_features_entry = AudioFeatures() - audio_features_entry.track = track - for key, val in response.items(): - if key not in useless_keys: - setattr(audio_features_entry, key, val) - audio_features_entry.save() - - # }}} get_audio_features # -# update_std_dev {{{ # - -def update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size): - """Calculates the standard deviation for a sample without storing all data points - - Args: - cur_mean: the current mean for N = (sample_size - 1) - cur_std_dev: the current standard deviation for N = (sample_size - 1) - new_data_point: a new data point - sample_size: sample size including the new data point - - Returns: - (new_mean, new_std_dev) - """ - # This is an implementation of Welford's method - # http://jonisalonen.com/2013/deriving-welfords-method-for-computing-variance/ - new_mean = ((sample_size - 1) * cur_mean + new_data_point) / sample_size - delta_variance = (new_data_point - new_mean) * (new_data_point - cur_mean) - new_std_dev = math.sqrt( - (math.pow(cur_std_dev, 2) * (sample_size - 2) + delta_variance) / ( - sample_size - 1 - )) - return new_mean, new_std_dev - -# }}} update_std_dev # - -# update_audio_feature_stats {{{ # - -def update_audio_feature_stats(feature, new_data_point, sample_size, library_stats): - """Updates the audio feature statistics in library_stats - - Args: - feature: the audio feature to be updated (string) - new_data_point: new data to update the stats with - sample_size: sample size including the new data point - library_stats Dictionary containing the data mined from user's Spotify library - - - Returns: - None - """ - # first time the feature is considered - if sample_size < 2: - library_stats['audio_features'][feature] = { - "average": new_data_point, - "std_dev": 0, - } - else: - cur_mean = library_stats['audio_features'][feature]['average'] - cur_std_dev = library_stats['audio_features'][feature]['std_dev'] - new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size) - - library_stats['audio_features'][feature] = { - "average": new_mean, - "std_dev": new_std_dev - } - -# }}} update_audio_feature_stats # - -# increase_nested_key {{{ # - -def increase_nested_key(top_key, nested_key, library_stats, amount=1): - """Increases count for the value of library_stats[top_key][nested_key]. Checks if nested_key exists already and takes - appropriate action. - - :top_key: First key of library_stats. - :nested_key: Key in top_key's dict for which we want to increase value of. - :library_stats: Dictionary containing the data mined from user's Spotify library - - :returns: None - - """ - if nested_key not in library_stats[top_key]: - library_stats[top_key][nested_key] = amount - else: - library_stats[top_key][nested_key] += amount - -# }}} increase_nested_key # - -# increase_artist_count {{{ # - -def increase_artist_count(headers, artist_name, artist_id, library_stats): - """Increases count for artist in library_stats and stores the artist_id. - - :headers: For making the API call. - :artist_name: Artist to increase count for. - :artist_id: The Spotify ID for the artist. - :library_stats: Dictionary containing the data mined from user's Spotify library - - :returns: None - - """ - if artist_name not in library_stats['artists']: - library_stats['artists'][artist_name] = {} - library_stats['artists'][artist_name]['count'] = 1 - library_stats['artists'][artist_name]['id'] = artist_id - else: - library_stats['artists'][artist_name]['count'] += 1 - -# }}} increase_artist_count # - -# update_popularity_stats {{{ # - -def update_popularity_stats(new_data_point, library_stats, sample_size): - """Updates the popularity statistics in library_stats - - Args: - new_data_point: new data to update the popularity stats with - library_stats: Dictionary containing data mined from user's Spotify library - sample_size: The sample size including the new data - - Returns: - None - """ - if sample_size < 2: - library_stats['popularity'] = { - "average": new_data_point, - "std_dev": 0, - } - else : - cur_mean_popularity = library_stats['popularity']['average'] - cur_popularity_stdev = library_stats['popularity']['std_dev'] - new_mean, new_std_dev = update_std_dev( - cur_mean_popularity, cur_popularity_stdev, new_data_point, sample_size) - library_stats['popularity'] = { - "average": new_mean, - "std_dev": new_std_dev, - } - -# }}} update_popularity_stats # - -# get_track_info {{{ # - -def get_track_info(track_dict, library_stats, sample_size): - """Get all the info from the track_dict directly returned by the API call in parse_library. - - :track_dict: Dict returned from the API call containing the track info. - :library_stats: Dictionary containing the data mined from user's Spotify library - :sample_size: The sample size so far including this track - - :returns: None - - """ - # popularity - update_popularity_stats(track_dict['popularity'], library_stats, sample_size) - - # year - year_released = track_dict['album']['release_date'].split('-')[0] - increase_nested_key('year_released', year_released, library_stats) - - # runtime - library_stats['total_runtime'] += float(track_dict['duration_ms']) / (1000 * 60) - -# }}} get_track_info # - -# update_artist_genre {{{ # - -def update_artist_genre(headers, artist_obj): - """Updates the top genre for an artist by querying the Spotify API - - :headers: For making the API call. - :artist_obj: the Artist object whose genre field will be updated - - :returns: None - - """ - artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_obj.artist_id, headers=headers).json() - # update genre for artist in database with top genre - if len(artist_response['genres']) > 0: - artist_obj.genre = artist_response['genres'][0] - artist_obj.save() - -# }}} # - -# get_top_genre {{{ # - +# WIP: is this being removed to redo genre data? def get_top_genre(headers, top_artist_id): """Updates the top genre for a track by querying the Spotify API @@ -401,7 +204,7 @@ def get_top_genre(headers, top_artist_id): else: return "undefined" -# }}} # +# add_artist_genres {{{ # def add_artist_genres(headers, artist_objs): """Adds genres to artist_objs and increases the count the respective Genre @@ -415,7 +218,6 @@ def add_artist_genres(headers, artist_objs): """ artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs]) - print(len(artist_objs), artist_ids) params = {'ids': artist_ids} artists_response = requests.get('https://api.spotify.com/v1/artists/', headers=headers, params=params).json()['artists'] @@ -430,60 +232,7 @@ def add_artist_genres(headers, artist_objs): artist_objs[i].genres.add(genre_obj) artist_objs[i].save() -# process_library_stats {{{ # - -def process_library_stats(library_stats): - """Processes library_stats into format more suitable for D3 consumption - - Args: - library_stats: Dictionary containing the data mined from user's Spotify library - - Returns: - A new dictionary that contains the data in library_stats, in a format more suitable for D3 consumption - """ - processed_library_stats = {} - for key in library_stats: - if key == 'artists' or key == 'genres' or key == 'year_released': - for inner_key in library_stats[key]: - if key not in processed_library_stats: - processed_library_stats[key] = [] - processed_item_key = '' # identifier key for each dict in the list - count = 0 - if 'artist' in key: - processed_item_key = 'name' - count = library_stats[key][inner_key]['count'] - elif 'genre' in key: - processed_item_key = 'genre' - count = library_stats[key][inner_key] - else: - processed_item_key = 'year' - count = library_stats[key][inner_key] - - processed_library_stats[key].append({ - processed_item_key: inner_key, - "count": count - }) - elif key == 'audio_features': - for audio_feature in library_stats[key]: - if 'audio_features' not in processed_library_stats: - processed_library_stats['audio_features'] = [] - processed_library_stats['audio_features'].append({ - 'feature': audio_feature, - 'average': library_stats[key][audio_feature]['average'], - 'std_dev': library_stats[key][audio_feature]['std_dev'] - }) - # TODO: Not sure about final form for 'popularity' - # elif key == 'popularity': - # processed_library_stats[key] = [] - # processed_library_stats[key].append({ - - # }) - elif key == 'num_songs' or key == 'total_runtime' or key == 'popularity': - processed_library_stats[key] = library_stats[key] - - return processed_library_stats - -# }}} process_library_stats # +# }}} add_artist_genres # # get_artists_in_genre {{{ # diff --git a/spotifyvis/views.py b/spotifyvis/views.py index d1a212b..79f9854 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -13,7 +13,7 @@ from datetime import datetime from django.shortcuts import render, redirect from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse from django.db.models import Count, Q -from .utils import parse_library, process_library_stats, get_artists_in_genre, update_track_genres +from .utils import parse_library, get_artists_in_genre, update_track_genres from .models import User, Track, AudioFeatures, Artist # }}} imports # @@ -42,7 +42,6 @@ def generate_random_string(length): # token_expired {{{ # - def token_expired(token_obtained_at, valid_for): """Returns True if token expired, False if otherwise @@ -119,6 +118,9 @@ def callback(request): # user_data {{{ # def user_data(request): + + # get user token {{{ # + token_obtained_at = datetime.strptime(request.session['token_obtained_at'], TIME_FORMAT) valid_for = int(request.session['valid_for']) @@ -133,6 +135,8 @@ def user_data(request): refresh_token_response = requests.post('https://accounts.spotify.com/api/token', data = req_body).json() request.session['access_token'] = refresh_token_response['access_token'] request.session['valid_for'] = refresh_token_response['expires_in'] + + # }}} get user token # auth_token_str = "Bearer " + request.session['access_token'] headers = { @@ -140,14 +144,18 @@ def user_data(request): } user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json() - request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model - # request.session['user_name'] = user_data_response['display_name'] + # store the user_id so it may be used to create model + request.session['user_id'] = user_data_response['id'] + # create user obj {{{ # + try: user = User.objects.get(user_id=user_data_response['id']) except User.DoesNotExist: user = User(user_id=user_data_response['id'], user_secret=generate_random_string(30)) user.save() + + # }}} create user obj # context = { 'id': user_data_response['id'], From 0c0520af8cf4ce51bfad965f457d8b0dc342641a Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 27 Jun 2018 02:06:17 -0400 Subject: [PATCH 47/49] Added most common genre to all tracks (closes #34) Resolved issue of there being multiple artists on a track and "manually created" an undefined genre for artists that don't have a genre. --- .../templates/spotifyvis/user_data.html | 5 -- spotifyvis/utils.py | 75 +++++++++++-------- spotifyvis/views.py | 3 +- 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/spotifyvis/templates/spotifyvis/user_data.html b/spotifyvis/templates/spotifyvis/user_data.html index 440d756..67c99db 100644 --- a/spotifyvis/templates/spotifyvis/user_data.html +++ b/spotifyvis/templates/spotifyvis/user_data.html @@ -17,10 +17,5 @@

    You are using an outdated browser. Please upgrade your browser to improve your experience.

    Logged in as {{ id }}

    - - diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index f8e2599..fae4439 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -12,10 +12,10 @@ import json # }}} imports # USER_TRACKS_LIMIT = 50 -# ARTIST_LIMIT = 50 -ARTIST_LIMIT = 25 -# FEATURES_LIMIT = 100 -FEATURES_LIMIT = 25 +ARTIST_LIMIT = 50 +FEATURES_LIMIT = 100 +# ARTIST_LIMIT = 25 +# FEATURES_LIMIT = 25 # parse_library {{{ # @@ -37,7 +37,7 @@ def parse_library(headers, tracks, user): features_queue = [] # iterate until hit requested num of tracks - for _ in range(0, tracks, USER_TRACKS_LIMIT): + for i in range(0, tracks, USER_TRACKS_LIMIT): payload['offset'] = str(offset) saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', headers=headers, @@ -80,6 +80,12 @@ def parse_library(headers, tracks, user): # }}} add audio features # + # temporary console logging + print("#{}-{}: {} - {}".format(offset + 1, + offset + USER_TRACKS_LIMIT, + track_obj.artists.first(), + track_obj.name)) + # calculates num_songs with offset + songs retrieved offset += USER_TRACKS_LIMIT @@ -111,11 +117,18 @@ def update_track_genres(user): """ user_tracks = Track.objects.filter(users__exact=user) for track in user_tracks: - track_artists = list(track.artists.all()) - if len(track_artists) == 1: - track.genre = track_artists[0].genres.all().order_by('-num_songs').first() - track.save() - # print(track_artists, track.genre) + # just using this variable to save another call to db + track_artists = track.artists.all() + # set genres to first artist's genres then find intersection with others + shared_genres = track_artists.first().genres.all() + for artist in track_artists: + shared_genres.intersection(artist.genres.all()) + + most_common_genre = shared_genres.order_by('-num_songs').first() + track.genre = most_common_genre if most_common_genre is not None \ + else "undefined" + track.save() + # print(track.name, track.genre) # }}} update_track_genres # @@ -186,23 +199,22 @@ def get_audio_features(headers, track_objs): # }}} get_audio_features # -# WIP: is this being removed to redo genre data? -def get_top_genre(headers, top_artist_id): - """Updates the top genre for a track by querying the Spotify API - - :headers: For making the API call. - :top_artist: The first artist's (listed in the track) Spotify ID. +def process_artist_genre(genre_name, artist_obj): + """Increase count for correspoding Genre object to genre_name and add that + Genre to artist_obj. - :returns: The first genre listed for the top_artist. + :genre_name: Name of genre. + :artist_obj: Artist object to add Genre object to. + :returns: None """ - artist_response = requests.get('https://api.spotify.com/v1/artists/' + - top_artist_id, headers=headers).json() - # pprint.pprint(artist_response) - if len(artist_response['genres']) > 0: - return artist_response['genres'][0] - else: - return "undefined" + genre_obj, created = Genre.objects.get_or_create(name=genre_name, + defaults={'num_songs':1}) + if not created: + genre_obj.num_songs = F('num_songs') + 1 + genre_obj.save() + artist_obj.genres.add(genre_obj) + artist_obj.save() # add_artist_genres {{{ # @@ -223,14 +235,11 @@ def add_artist_genres(headers, artist_objs): headers=headers, params=params).json()['artists'] # pprint.pprint(artists_response) for i in range(len(artist_objs)): - for genre in artists_response[i]['genres']: - genre_obj, created = Genre.objects.get_or_create(name=genre, - defaults={'num_songs':1}) - if not created: - genre_obj.num_songs = F('num_songs') +1 - genre_obj.save() - artist_objs[i].genres.add(genre_obj) - artist_objs[i].save() + if len(artists_response[i]['genres']) == 0: + process_artist_genre("undefined", artist_objs[i]) + else: + for genre in artists_response[i]['genres']: + process_artist_genre(genre, artist_objs[i]) # }}} add_artist_genres # @@ -255,6 +264,8 @@ def get_artists_in_genre(user, genre, max_songs): processed_artist_counts = {} songs_added = 0 for artist in artist_counts: + # hacky way to not have total count overflow due to there being multiple + # artists on a track if songs_added + artist.num_songs <= max_songs: processed_artist_counts[artist.name] = artist.num_songs songs_added += artist.num_songs diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 79f9854..bf5fcc7 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -19,7 +19,7 @@ from .models import User, Track, AudioFeatures, Artist # }}} imports # TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' -TRACKS_TO_QUERY = 50 +TRACKS_TO_QUERY = 200 # generate_random_string {{{ # @@ -237,6 +237,7 @@ def get_genre_data(request, user_secret): for genre_dict in genre_counts: genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre'], genre_dict['num_songs']) + print("*** Genre Breakdown ***") pprint.pprint(list(genre_counts)) return JsonResponse(data=list(genre_counts), safe=False) From 3aa49cc4e153039503ec0cd3cde52288761d0d5b Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 27 Jun 2018 03:34:35 -0400 Subject: [PATCH 48/49] Resolves #35 --- .../{test_db.html => genre_graph.html} | 0 spotifyvis/templates/spotifyvis/index.html | 8 +------- spotifyvis/templates/spotifyvis/logged_in.html | 9 +++++++-- spotifyvis/urls.py | 6 ++++-- spotifyvis/utils.py | 2 +- spotifyvis/views.py | 17 +++++++++++------ 6 files changed, 24 insertions(+), 18 deletions(-) rename spotifyvis/templates/spotifyvis/{test_db.html => genre_graph.html} (100%) diff --git a/spotifyvis/templates/spotifyvis/test_db.html b/spotifyvis/templates/spotifyvis/genre_graph.html similarity index 100% rename from spotifyvis/templates/spotifyvis/test_db.html rename to spotifyvis/templates/spotifyvis/genre_graph.html diff --git a/spotifyvis/templates/spotifyvis/index.html b/spotifyvis/templates/spotifyvis/index.html index 28a9301..9f6e27a 100644 --- a/spotifyvis/templates/spotifyvis/index.html +++ b/spotifyvis/templates/spotifyvis/index.html @@ -21,13 +21,7 @@

    spotify-lib-vis

    Scan Library - Test DB -
    - -
    -
      - -
    + Admin Graphs
    diff --git a/spotifyvis/templates/spotifyvis/logged_in.html b/spotifyvis/templates/spotifyvis/logged_in.html index a3d9017..849a59b 100644 --- a/spotifyvis/templates/spotifyvis/logged_in.html +++ b/spotifyvis/templates/spotifyvis/logged_in.html @@ -5,8 +5,13 @@ Logged In + - Audio Features +

    {{ user_id }}'s Graphs

    + Audio Features + Genres - \ No newline at end of file + diff --git a/spotifyvis/urls.py b/spotifyvis/urls.py index f5bc82b..868f86b 100644 --- a/spotifyvis/urls.py +++ b/spotifyvis/urls.py @@ -8,9 +8,11 @@ urlpatterns = [ path('login', login, name='login'), path('callback', callback, name='callback'), path('user_data', user_data, name='user_data'), - path('test_db', test_db, name='test_db'), + path('admin_graphs', admin_graphs, name='admin_graphs'), path('user_artists/', get_artist_data, name='get_artist_data'), - path('user_genres/', get_genre_data, name='get_genre_data'), + path('api/user_genres/', get_genre_data, name='get_genre_data'), + path('graphs/genre/', display_genre_graph, + name='display_genre_graph'), path('audio_features/', audio_features, name='audio_features'), path('audio_features//', get_audio_feature_data, name='get_audio_feature_data'), diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index bccd7d0..9201a32 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -63,7 +63,7 @@ def parse_library(headers, tracks, user): # }}} add artists # - # WIP: get most common genre + # TODO: fix this, don't need any more top_genre = "" track_obj, track_created = save_track_obj(track_dict['track'], track_artists, top_genre, user) diff --git a/spotifyvis/views.py b/spotifyvis/views.py index d52b391..d48acef 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -161,6 +161,7 @@ def user_data(request): # }}} create user obj # context = { + 'user_id': user.user_id, 'user_secret': user.user_secret, } @@ -169,21 +170,18 @@ def user_data(request): # }}} user_data # -# test_db {{{ # - -def test_db(request): +def admin_graphs(request): """TODO """ user_id = "polarbier" # user_id = "chrisshyi13" user_obj = User.objects.get(user_id=user_id) context = { + 'user_id': user_id, 'user_secret': user_obj.user_secret, } update_track_genres(user_obj) - return render(request, 'spotifyvis/test_db.html', context) - -# }}} test_db # + return render(request, 'spotifyvis/logged_in.html', context) # get_artist_data {{{ # @@ -199,6 +197,13 @@ def get_artist_data(request, user_secret): # }}} get_artist_data # +def display_genre_graph(request, client_secret): + user = User.objects.get(user_secret=client_secret) + context = { + 'user_secret': client_secret, + } + return render(request, "spotifyvis/genre_graph.html", context) + def audio_features(request, client_secret): user = User.objects.get(user_secret=client_secret) context = { From 2b98398b6c09d6d6675ad2c688611ede34dd2ab4 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 27 Jun 2018 06:12:55 -0400 Subject: [PATCH 49/49] Fixed bug in get_artists_in_genre Artist would show up in "undefined" genre even when they don't have that genre. Likely due to changing genre to be a model so updated code to adjust to that. --- .gitignore | 2 +- spotifyvis/utils.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 5a848c6..c492bbd 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ db.sqlite3 api-keys.sh Pipfile *.txt -graph.js +scrap.py diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 9201a32..668c274 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -122,11 +122,14 @@ def update_track_genres(user): # set genres to first artist's genres then find intersection with others shared_genres = track_artists.first().genres.all() for artist in track_artists: - shared_genres.intersection(artist.genres.all()) + shared_genres = shared_genres.intersection(artist.genres.all()) + shared_genres = shared_genres.order_by('-num_songs') - most_common_genre = shared_genres.order_by('-num_songs').first() + undefined_genre_obj = Genre.objects.get(name="undefined") + most_common_genre = shared_genres.first() if shared_genres.first() is \ + not undefined_genre_obj else shared_genres[1] track.genre = most_common_genre if most_common_genre is not None \ - else "undefined" + else undefined_genre_obj track.save() # print(track.name, track.genre) @@ -256,8 +259,9 @@ def get_artists_in_genre(user, genre, max_songs): :returns: dict of artists in the genre along with the number of songs they have. """ + genre_obj = Genre.objects.get(name=genre) artist_counts = (Artist.objects.filter(track__users=user) - .filter(track__genre=genre) + .filter(genres=genre_obj) .annotate(num_songs=Count('track', distinct=True)) .order_by('-num_songs') )