From 4b263532d167279a003cfa2fd0c871d3ad5d89a1 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Sun, 20 May 2018 18:40:22 -0400 Subject: [PATCH 1/9] Added missing space on line #222 --- spotifyvis/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spotifyvis/views.py b/spotifyvis/views.py index e580c96..a2749da 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -219,7 +219,7 @@ def update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size): Returns: (new_mean, new_std_dev) """ - # This is an implementationof Welford's method + # This is an implementation of Welford's method # http://jonisalonen.com/2013/deriving-welfords-method-for-computing-variance/ new_mean = ((sample_size - 1) * cur_mean + new_data_point) / sample_size delta_variance = (new_data_point - new_mean) * (new_data_point - cur_mean) From c6c4897ce0cd6cf2a994d81853e910e56c4ea3bb Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Mon, 21 May 2018 11:55:32 -0400 Subject: [PATCH 2/9] Added four tests for update_std_dev() --- spotifyvis/tests.py | 53 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/spotifyvis/tests.py b/spotifyvis/tests.py index 7ce503c..8a9a2d5 100644 --- a/spotifyvis/tests.py +++ b/spotifyvis/tests.py @@ -1,3 +1,54 @@ from django.test import TestCase - +from .views import update_std_dev +import math # Create your tests here. + +class UpdateStdDevTest(TestCase): + + def test_two_data_points(self): + """ + tests if update_std_dev behaves correctly for two data points + """ + cur_mean = 5 + cur_std_dev = 0 + + new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 10, 2) + + self.assertTrue(math.isclose(new_mean, 7.5, rel_tol=0.01)) + self.assertTrue(math.isclose(new_std_dev, 3.5355, rel_tol=0.01)) + + def test_three_data_points(self): + """ + tests if update_std_dev behaves correctly for three data points + """ + cur_mean = 7.5 + cur_std_dev = 3.5355 + + new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 15, 3) + + self.assertTrue(math.isclose(new_mean, 10, rel_tol=0.01)) + self.assertTrue(math.isclose(new_std_dev, 5, rel_tol=0.01)) + + def test_four_data_points(self): + """ + tests if update_std_dev behaves correctly for four data points + """ + cur_mean = 10 + cur_std_dev = 5 + + new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 20, 4) + self.assertTrue(math.isclose(new_mean, 12.5, rel_tol=0.01)) + self.assertTrue(math.isclose(new_std_dev, 6.455, rel_tol=0.01)) + + + + def test_five_data_points(self): + """ + tests if update_std_dev behaves correctly for five data points + """ + cur_mean = 12.5 + cur_std_dev = 6.455 + + new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 63, 5) + self.assertTrue(math.isclose(new_mean, 22.6, rel_tol=0.01)) + self.assertTrue(math.isclose(new_std_dev, 23.2658, rel_tol=0.01)) \ No newline at end of file From 3a733e18b3fa470aadc864b8edfeafe196b1cf8c Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Mon, 21 May 2018 12:05:43 -0400 Subject: [PATCH 3/9] Added one more test for update_std_dev() --- spotifyvis/tests.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/spotifyvis/tests.py b/spotifyvis/tests.py index 8a9a2d5..b88c1e2 100644 --- a/spotifyvis/tests.py +++ b/spotifyvis/tests.py @@ -17,6 +17,7 @@ class UpdateStdDevTest(TestCase): self.assertTrue(math.isclose(new_mean, 7.5, rel_tol=0.01)) self.assertTrue(math.isclose(new_std_dev, 3.5355, rel_tol=0.01)) + def test_three_data_points(self): """ tests if update_std_dev behaves correctly for three data points @@ -28,7 +29,8 @@ class UpdateStdDevTest(TestCase): self.assertTrue(math.isclose(new_mean, 10, rel_tol=0.01)) self.assertTrue(math.isclose(new_std_dev, 5, rel_tol=0.01)) - + + def test_four_data_points(self): """ tests if update_std_dev behaves correctly for four data points @@ -41,7 +43,6 @@ class UpdateStdDevTest(TestCase): self.assertTrue(math.isclose(new_std_dev, 6.455, rel_tol=0.01)) - def test_five_data_points(self): """ tests if update_std_dev behaves correctly for five data points @@ -51,4 +52,16 @@ class UpdateStdDevTest(TestCase): new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 63, 5) self.assertTrue(math.isclose(new_mean, 22.6, rel_tol=0.01)) - self.assertTrue(math.isclose(new_std_dev, 23.2658, rel_tol=0.01)) \ No newline at end of file + self.assertTrue(math.isclose(new_std_dev, 23.2658, rel_tol=0.01)) + + + def test_sixteen_data_points(self): + """ + tests if update_std_dev behaves correctly for sixteen data points + """ + cur_mean = 0.4441 + cur_std_dev = 0.2855 + + new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 0.7361, 16) + self.assertTrue(math.isclose(new_mean, 0.4624, rel_tol=0.01)) + self.assertTrue(math.isclose(new_std_dev, 0.2853, rel_tol=0.01)) \ No newline at end of file From c9de0395a4bbe5bc27966bd145b0b782aa51dd41 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Tue, 22 May 2018 18:11:56 -0400 Subject: [PATCH 4/9] Refactor helper functions in views.py Refactored helper functions in views.py into utils.py. Instead of having a global dictionary object for user library data, helper functions now mutate a dictionary passed in as a parameter. --- spotifyvis/utils.py | 209 ++++++++++++++++++++++++++++++++++++++++++++ spotifyvis/views.py | 200 +----------------------------------------- 2 files changed, 213 insertions(+), 196 deletions(-) create mode 100644 spotifyvis/utils.py diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py new file mode 100644 index 0000000..0fad798 --- /dev/null +++ b/spotifyvis/utils.py @@ -0,0 +1,209 @@ +import requests +import math +import pprint + +# parse_library {{{ # + +def parse_library(headers, tracks, library_stats): + """Scans user's library for certain number of tracks to update library_stats with. + + :headers: For API call. + :tracks: Number of tracks to get from user's library. + :library_stats: Dictionary containing the data mined from user's library + + :returns: None + + """ + # TODO: implement importing entire library with 0 as tracks param + # number of tracks to get with each call + limit = 5 + # keeps track of point to get songs from + offset = 0 + payload = {'limit': str(limit)} + for _ in range(0, tracks, limit): + payload['offset'] = str(offset) + saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', headers=headers, params=payload).json() + num_samples = offset + for track_dict in saved_tracks_response['items']: + # Track the number of samples for calculating + # audio feature averages and standard deviations on the fly + num_samples += 1 + get_track_info(track_dict['track'], library_stats) + # get_genre(headers, track_dict['track']['album']['id']) + audio_features_dict = get_audio_features(headers, track_dict['track']['id']) + for feature, feature_data in audio_features_dict.items(): + update_audio_feature_stats(feature, feature_data, num_samples, library_stats) + for artist_dict in track_dict['track']['artists']: + increase_artist_count(headers, artist_dict['name'], artist_dict['id'], library_stats) + # calculates num_songs with offset + songs retrieved + library_stats['num_songs'] = offset + len(saved_tracks_response['items']) + offset += limit + calculate_genres_from_artists(headers, library_stats) + pprint.pprint(library_stats) + +# }}} parse_library # + +def get_audio_features(headers, track_id): + """Returns the audio features of a soundtrack + + Args: + headers: headers containing the API token + track_id: the id of the soundtrack, needed to query the Spotify API + + Returns: + A dictionary with the features as its keys + """ + + response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json() + features_dict = {} + + # Data that we don't need + useless_keys = [ + "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", + ] + for key, val in response.items(): + if key not in useless_keys: + features_dict[key] = val + + return features_dict + + +def update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size): + """Calculates the standard deviation for a sample without storing all data points + + Args: + cur_mean: the current mean for N = (sample_size - 1) + cur_std_dev: the current standard deviation for N = (sample_size - 1) + new_data_point: a new data point + sample_size: sample size including the new data point + + Returns: + (new_mean, new_std_dev) + """ + # This is an implementation of Welford's method + # http://jonisalonen.com/2013/deriving-welfords-method-for-computing-variance/ + new_mean = ((sample_size - 1) * cur_mean + new_data_point) / sample_size + delta_variance = (new_data_point - new_mean) * (new_data_point - cur_mean) + new_std_dev = math.sqrt( + (math.pow(cur_std_dev, 2) * (sample_size - 2) + delta_variance) / ( + sample_size - 1 + )) + return new_mean, new_std_dev + + +def update_audio_feature_stats(feature, new_data_point, sample_size, library_stats): + """Updates the audio feature statistics in library_stats + + Args: + feature: the audio feature to be updated (string) + new_data_point: new data to update the stats with + sample_size: sample size including the new data point + library_stats Dictionary containing the data mined from user's Spotify library + + + Returns: + None + """ + # first time the feature is considered + if sample_size < 2: + library_stats['audio_features'][feature] = { + "average": new_data_point, + "std_dev": 0, + } + else: + cur_mean = library_stats['audio_features'][feature]['average'] + cur_std_dev = library_stats['audio_features'][feature]['std_dev'] + new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size) + + library_stats['audio_features'][feature]['average'] = new_mean + library_stats['audio_features'][feature]['std_dev'] = new_std_dev + + +# increase_nested_key {{{ # + +def increase_nested_key(top_key, nested_key, library_stats, amount=1): + """Increases count for the value of library_stats[top_key][nested_key]. Checks if nested_key exists already and takes + appropriate action. + + :top_key: First key of library_stats. + :nested_key: Key in top_key's dict for which we want to increase value of. + :library_stats: Dictionary containing the data mined from user's Spotify library + + :returns: None + + """ + if nested_key not in library_stats[top_key]: + library_stats[top_key][nested_key] = amount + else: + library_stats[top_key][nested_key] += amount + +# }}} increase_nested_key # + +# increase_artist_count {{{ # + +def increase_artist_count(headers, artist_name, artist_id, library_stats): + """Increases count for artist in library_stats and stores the artist_id. + + :headers: For making the API call. + :artist_name: Artist to increase count for. + :artist_id: The Spotify ID for the artist. + :library_stats: Dictionary containing the data mined from user's Spotify library + + :returns: None + + """ + if artist_name not in library_stats['artists']: + library_stats['artists'][artist_name] = {} + library_stats['artists'][artist_name]['count'] = 1 + library_stats['artists'][artist_name]['id'] = artist_id + else: + library_stats['artists'][artist_name]['count'] += 1 + +# }}} increase_artist_count # + +# get_track_info {{{ # + +def get_track_info(track_dict, library_stats): + """Get all the info from the track_dict directly returned by the API call in parse_library. + + :track_dict: Dict returned from the API call containing the track info. + :library_stats: Dictionary containing the data mined from user's Spotify library + + :returns: None + + """ + # popularity + library_stats['popularity'].append(track_dict['popularity']) + + # year + year_released = track_dict['album']['release_date'].split('-')[0] + increase_nested_key('year_released', year_released, library_stats) + + # artist + # artist_names = [artist['name'] for artist in track_dict['artists']] + # for artist_name in artist_names: + # increase_nested_key('artists', artist_name) + + # runtime + library_stats['total_runtime'] += float(track_dict['duration_ms']) / 60 + +# }}} get_track_info # + +# calculate_genres_from_artists {{{ # + +def calculate_genres_from_artists(headers, library_stats): + """Tallies up genre counts based on artists in library_stats. + + :headers: For making the API call. + :library_stats: Dictionary containing the data mined from user's Spotify library + + :returns: None + + """ + for artist_entry in library_stats['artists'].values(): + artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_entry['id'], headers=headers).json() + # increase each genre count by artist count + for genre in artist_response['genres']: + increase_nested_key('genres', genre, library_stats, artist_entry['count']) + +# }}} calculate_genres_from_artists # \ No newline at end of file diff --git a/spotifyvis/views.py b/spotifyvis/views.py index a2749da..0e5816b 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -8,6 +8,7 @@ import urllib import json import pprint from datetime import datetime +from .utils import * TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' library_stats = {"audio_features":{}, "genres":{}, "year_released":{}, "artists":{}, "num_songs":0, "popularity":[], "total_runtime":0} @@ -138,201 +139,8 @@ def user_data(request): } tracks_to_query = 5 - parse_library(headers, tracks_to_query) + library_stats = {"audio_features":{}, "genres":{}, "year_released":{}, "artists":{}, "num_songs":0, "popularity":[], "total_runtime":0} + parse_library(headers, tracks_to_query, library_stats) return render(request, 'spotifyvis/user_data.html', context) -# }}} user_data # - -# parse_library {{{ # - -def parse_library(headers, tracks): - """Scans user's library for certain number of tracks to update library_stats with. - - :headers: For API call. - :tracks: Number of tracks to get from user's library. - :returns: None - - """ - # TODO: implement importing entire library with 0 as tracks param - # number of tracks to get with each call - limit = 5 - # keeps track of point to get songs from - offset = 0 - payload = {'limit': str(limit)} - for _ in range(0, tracks, limit): - payload['offset'] = str(offset) - saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', headers=headers, params=payload).json() - num_samples = offset - for track_dict in saved_tracks_response['items']: - # Track the number of samples for calculating - # audio feature averages and standard deviations on the fly - num_samples += 1 - get_track_info(track_dict['track']) - # get_genre(headers, track_dict['track']['album']['id']) - audio_features_dict = get_audio_features(headers, track_dict['track']['id']) - for feature, feature_data in audio_features_dict.items(): - update_audio_feature_stats(feature, feature_data, num_samples) - for artist_dict in track_dict['track']['artists']: - increase_artist_count(headers, artist_dict['name'], artist_dict['id']) - # calculates num_songs with offset + songs retrieved - library_stats['num_songs'] = offset + len(saved_tracks_response['items']) - offset += limit - calculate_genres_from_artists(headers) - pprint.pprint(library_stats) - -# }}} parse_library # - -def get_audio_features(headers, track_id): - """Returns the audio features of a soundtrack - - Args: - headers: headers containing the API token - track_id: the id of the soundtrack, needed to query the Spotify API - - Returns: - A dictionary with the features as its keys - """ - - response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json() - features_dict = {} - - # Data that we don't need - useless_keys = [ - "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", - ] - for key, val in response.items(): - if key not in useless_keys: - features_dict[key] = val - - return features_dict - - -def update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size): - """Calculates the standard deviation for a sample without storing all data points - - Args: - cur_mean: the current mean for N = (sample_size - 1) - cur_std_dev: the current standard deviation for N = (sample_size - 1) - new_data_point: a new data point - sample_size: sample size including the new data point - - Returns: - (new_mean, new_std_dev) - """ - # This is an implementation of Welford's method - # http://jonisalonen.com/2013/deriving-welfords-method-for-computing-variance/ - new_mean = ((sample_size - 1) * cur_mean + new_data_point) / sample_size - delta_variance = (new_data_point - new_mean) * (new_data_point - cur_mean) - new_std_dev = math.sqrt( - (math.pow(cur_std_dev, 2) * (sample_size - 2) + delta_variance) / ( - sample_size - 1 - )) - return new_mean, new_std_dev - - -def update_audio_feature_stats(feature, new_data_point, sample_size): - """Updates the audio feature statistics in library_stats - - Args: - feature: the audio feature to be updated (string) - new_data_point: new data to update the stats with - sample_size: sample size including the new data point - - Returns: - None - """ - # first time the feature is considered - if sample_size < 2: - library_stats['audio_features'][feature] = { - "average": new_data_point, - "std_dev": 0, - } - else: - cur_mean = library_stats['audio_features'][feature]['average'] - cur_std_dev = library_stats['audio_features'][feature]['std_dev'] - new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size) - - library_stats['audio_features'][feature]['average'] = new_mean - library_stats['audio_features'][feature]['std_dev'] = new_std_dev - - -# increase_nested_key {{{ # - -def increase_nested_key(top_key, nested_key, amount=1): - """Increases count for the value of library_stats[top_key][nested_key]. Checks if nested_key exists already and takes - appropriate action. - - :top_key: First key of library_stats. - :nested_key: Key in top_key's dict for which we want to increase value of. - :returns: None - - """ - if nested_key not in library_stats[top_key]: - library_stats[top_key][nested_key] = amount - else: - library_stats[top_key][nested_key] += amount - -# }}} increase_nested_key # - -# increase_artist_count {{{ # - -def increase_artist_count(headers, artist_name, artist_id): - """Increases count for artist in library_stats and stores the artist_id. - - :headers: For making the API call. - :artist_name: Artist to increase count for. - :artist_id: The Spotify ID for the artist. - :returns: None - - """ - if artist_name not in library_stats['artists']: - library_stats['artists'][artist_name] = {} - library_stats['artists'][artist_name]['count'] = 1 - library_stats['artists'][artist_name]['id'] = artist_id - else: - library_stats['artists'][artist_name]['count'] += 1 - -# }}} increase_artist_count # - -# get_track_info {{{ # - -def get_track_info(track_dict): - """Get all the info from the track_dict directly returned by the API call in parse_library. - - :track_dict: Dict returned from the API call containing the track info. - :returns: None - - """ - # popularity - library_stats['popularity'].append(track_dict['popularity']) - - # year - year_released = track_dict['album']['release_date'].split('-')[0] - increase_nested_key('year_released', year_released) - - # artist - # artist_names = [artist['name'] for artist in track_dict['artists']] - # for artist_name in artist_names: - # increase_nested_key('artists', artist_name) - - # runtime - library_stats['total_runtime'] += float(track_dict['duration_ms']) / 60 - -# }}} get_track_info # - -# calculate_genres_from_artists {{{ # - -def calculate_genres_from_artists(headers): - """Tallies up genre counts based on artists in library_stats. - - :headers: For making the API call. - :returns: None - - """ - for artist_entry in library_stats['artists'].values(): - artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_entry['id'], headers=headers).json() - # increase each genre count by artist count - for genre in artist_response['genres']: - increase_nested_key('genres', genre, artist_entry['count']) - -# }}} calculate_genres_from_artists # +# }}} user_data # \ No newline at end of file From 46df571855b765c926c3e59c08f7cc13f3d01829 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Tue, 22 May 2018 20:25:29 -0400 Subject: [PATCH 5/9] Compute the average and std_dev for popularity Modified the structure and code in utils.py so that library_stats now has a record of the mean and standard deviation of popularity. --- spotifyvis/utils.py | 43 ++++++++++++++++++++++++++++++++++++------- spotifyvis/views.py | 13 ++++++++++++- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 0fad798..235c958 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -28,7 +28,7 @@ def parse_library(headers, tracks, library_stats): # Track the number of samples for calculating # audio feature averages and standard deviations on the fly num_samples += 1 - get_track_info(track_dict['track'], library_stats) + get_track_info(track_dict['track'], library_stats, num_samples) # get_genre(headers, track_dict['track']['album']['id']) audio_features_dict = get_audio_features(headers, track_dict['track']['id']) for feature, feature_data in audio_features_dict.items(): @@ -115,8 +115,10 @@ def update_audio_feature_stats(feature, new_data_point, sample_size, library_sta cur_std_dev = library_stats['audio_features'][feature]['std_dev'] new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size) - library_stats['audio_features'][feature]['average'] = new_mean - library_stats['audio_features'][feature]['std_dev'] = new_std_dev + library_stats['audio_features'][feature] = { + "average": new_mean, + "std_dev": new_std_dev + } # increase_nested_key {{{ # @@ -161,20 +163,47 @@ def increase_artist_count(headers, artist_name, artist_id, library_stats): # }}} increase_artist_count # +def update_popularity_stats(new_data_point, library_stats, sample_size): + """Updates the popularity statistics in library_stats + + Args: + new_data_point: new data to update the popularity stats with + library_stats: Dictionary containing data mined from user's Spotify library + sample_size: The sample size including the new data + + Returns: + None + """ + if sample_size < 2: + library_stats['popularity'] = { + "average": new_data_point, + "std_dev": 0, + } + else : + cur_mean_popularity = library_stats['popularity']['average'] + cur_popularity_stdev = library_stats['popularity']['std_dev'] + new_mean, new_std_dev = update_std_dev( + cur_mean_popularity, cur_popularity_stdev, new_data_point, sample_size) + library_stats['popularity'] = { + "average": new_mean, + "std_dev": new_std_dev, + } + # get_track_info {{{ # -def get_track_info(track_dict, library_stats): +def get_track_info(track_dict, library_stats, sample_size): """Get all the info from the track_dict directly returned by the API call in parse_library. :track_dict: Dict returned from the API call containing the track info. :library_stats: Dictionary containing the data mined from user's Spotify library + :sample_size: The sample size so far including this track :returns: None """ - # popularity - library_stats['popularity'].append(track_dict['popularity']) - + # popularity + update_popularity_stats(track_dict['popularity'], library_stats, sample_size) + # year year_released = track_dict['album']['release_date'].split('-')[0] increase_nested_key('year_released', year_released, library_stats) diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 0e5816b..012a80e 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -139,7 +139,18 @@ def user_data(request): } tracks_to_query = 5 - library_stats = {"audio_features":{}, "genres":{}, "year_released":{}, "artists":{}, "num_songs":0, "popularity":[], "total_runtime":0} + library_stats = { + "audio_features":{}, + "genres":{}, + "year_released":{}, + "artists":{}, + "num_songs": 0, + "popularity": { + "average": 0, + "std_dev": 0, + }, + "total_runtime": 0 + } parse_library(headers, tracks_to_query, library_stats) return render(request, 'spotifyvis/user_data.html', context) From e01331f9515f0bb40fe87b1087828499fdcba0a2 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Tue, 22 May 2018 21:22:24 -0400 Subject: [PATCH 6/9] Process library_stats library_stats is now processed into a format that's more easily consumed by D3. Need to finalize format for popularity data. --- spotifyvis/utils.py | 55 +++++++++++++++++++++++++++++++++++++++++++-- spotifyvis/views.py | 6 ++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 235c958..279e4ed 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -214,7 +214,7 @@ def get_track_info(track_dict, library_stats, sample_size): # increase_nested_key('artists', artist_name) # runtime - library_stats['total_runtime'] += float(track_dict['duration_ms']) / 60 + library_stats['total_runtime'] += float(track_dict['duration_ms']) / (1000 * 60) # }}} get_track_info # @@ -235,4 +235,55 @@ def calculate_genres_from_artists(headers, library_stats): for genre in artist_response['genres']: increase_nested_key('genres', genre, library_stats, artist_entry['count']) -# }}} calculate_genres_from_artists # \ No newline at end of file +# }}} calculate_genres_from_artists # + +def process_library_stats(library_stats): + """Processes library_stats into format more suitable for D3 consumption + + Args: + library_stats: Dictionary containing the data mined from user's Spotify library + + Returns: + A new dictionary that contains the data in library_stats, in a format more suitable for D3 consumption + """ + processed_library_stats = {} + for key in library_stats: + if key == 'artists' or key == 'genres' or key == 'year_released': + for inner_key in library_stats[key]: + if key not in processed_library_stats: + processed_library_stats[key] = [] + processed_item_key = '' # identifier key for each dict in the list + count = 0 + if 'artist' in key: + processed_item_key = 'name' + count = library_stats[key][inner_key]['count'] + elif 'genre' in key: + processed_item_key = 'genre' + count = library_stats[key][inner_key] + else: + processed_item_key = 'year' + count = library_stats[key][inner_key] + + processed_library_stats[key].append({ + processed_item_key: inner_key, + "count": count + }) + elif key == 'audio_features': + for audio_feature in library_stats[key]: + if 'audio_features' not in processed_library_stats: + processed_library_stats['audio_features'] = [] + processed_library_stats['audio_features'].append({ + 'feature': audio_feature, + 'average': library_stats[key][audio_feature]['average'], + 'std_dev': library_stats[key][audio_feature]['std_dev'] + }) + # TODO: Not sure about final form for 'popularity' + # elif key == 'popularity': + # processed_library_stats[key] = [] + # processed_library_stats[key].append({ + + # }) + elif key == 'num_songs' or key == 'total_runtime' or key == 'popularity': + processed_library_stats[key] = library_stats[key] + + return processed_library_stats diff --git a/spotifyvis/views.py b/spotifyvis/views.py index 012a80e..cfdf114 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -8,7 +8,7 @@ import urllib import json import pprint from datetime import datetime -from .utils import * +from .utils import parse_library, process_library_stats TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' library_stats = {"audio_features":{}, "genres":{}, "year_released":{}, "artists":{}, "num_songs":0, "popularity":[], "total_runtime":0} @@ -152,6 +152,10 @@ def user_data(request): "total_runtime": 0 } parse_library(headers, tracks_to_query, library_stats) + processed_library_stats = process_library_stats(library_stats) + print("================================================") + print("Processed data follows\n") + pprint.pprint(processed_library_stats) return render(request, 'spotifyvis/user_data.html', context) # }}} user_data # \ No newline at end of file From a508337363df67eeb0e7eb90fd1a7750aa33b6c8 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Sat, 2 Jun 2018 23:14:47 -0400 Subject: [PATCH 7/9] Fix import in tests.py tests.py was importing update_std_dev() from views.py (where the function was prior to the refactor), now fixed. --- spotifyvis/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spotifyvis/tests.py b/spotifyvis/tests.py index b88c1e2..ed439b4 100644 --- a/spotifyvis/tests.py +++ b/spotifyvis/tests.py @@ -1,5 +1,5 @@ from django.test import TestCase -from .views import update_std_dev +from .utils import update_std_dev import math # Create your tests here. From f52e88f83a375c5592643749878b1599469d64d8 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Sun, 3 Jun 2018 16:04:26 -0400 Subject: [PATCH 8/9] Start implementing AJAX Problem with cross origin requests, need to do more research. --- spotifyvis/static/spotifyvis/scripts/index.js | 42 +++++++++++++++++++ spotifyvis/templates/spotifyvis/index.html | 14 ++++++- 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 spotifyvis/static/spotifyvis/scripts/index.js diff --git a/spotifyvis/static/spotifyvis/scripts/index.js b/spotifyvis/static/spotifyvis/scripts/index.js new file mode 100644 index 0000000..cfac1d1 --- /dev/null +++ b/spotifyvis/static/spotifyvis/scripts/index.js @@ -0,0 +1,42 @@ +document.getElementById("login-btn").addEventListener("click", function() { + let httpRequest = new XMLHttpRequest(); + + /* + * Handler for the response + */ + httpRequest.onreadystatechange = function() { + if (httpRequest.readyState === XMLHttpRequest.DONE) { + if (httpRequest.status === 200) { + // hide the login button + document.getElementById('login').setAttribute("display", "none"); + + let responseData = JSON.parse(httpRequest.responseText); + let dataList = document.getElementById("data-list"); + + + for (let key in responseData) { + let newLi = document.createElement("li"); + let innerList = document.createElement("ul"); + + let dataLabel = document.createElement("li"); + dataLabel.innerText = key; + + let dataValue = document.createElement("li"); + dataValue.innerText = responseData[key]; + + innerList.appendChild(dataLabel); + innerList.appendChild(dataValue); + + newLi.appendChild(innerList); + dataList.appendChild(newLi); + } + } else { + alert("There was a problem with the login request, please try again!"); + } + } + } + + httpRequest.open('GET', '/login', true); + httpRequest.send(); +}); + diff --git a/spotifyvis/templates/spotifyvis/index.html b/spotifyvis/templates/spotifyvis/index.html index e410bf5..32d6db9 100644 --- a/spotifyvis/templates/spotifyvis/index.html +++ b/spotifyvis/templates/spotifyvis/index.html @@ -1,4 +1,5 @@ +{% load static %} User Login @@ -11,14 +12,25 @@ width: 500px; } +

This is an example of the Authorization Code flow

- Log in with Spotify + Log In (Original) + +
+ +
+
    + +
+ + + From 003c630dd4da143b731f4a5b596d9999ff278666 Mon Sep 17 00:00:00 2001 From: Chris Shyi Date: Sun, 3 Jun 2018 16:54:39 -0400 Subject: [PATCH 9/9] Fix audio feature analysis missing bug Added error checking for missing audio features in the Spotify database. --- spotifyvis/utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 279e4ed..9e537dc 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -27,12 +27,13 @@ def parse_library(headers, tracks, library_stats): for track_dict in saved_tracks_response['items']: # Track the number of samples for calculating # audio feature averages and standard deviations on the fly - num_samples += 1 get_track_info(track_dict['track'], library_stats, num_samples) # get_genre(headers, track_dict['track']['album']['id']) audio_features_dict = get_audio_features(headers, track_dict['track']['id']) - for feature, feature_data in audio_features_dict.items(): - update_audio_feature_stats(feature, feature_data, num_samples, library_stats) + if len(audio_features_dict) != 0: + num_samples += 1 + for feature, feature_data in audio_features_dict.items(): + update_audio_feature_stats(feature, feature_data, num_samples, library_stats) for artist_dict in track_dict['track']['artists']: increase_artist_count(headers, artist_dict['name'], artist_dict['id'], library_stats) # calculates num_songs with offset + songs retrieved @@ -51,10 +52,13 @@ def get_audio_features(headers, track_id): track_id: the id of the soundtrack, needed to query the Spotify API Returns: - A dictionary with the features as its keys + A dictionary with the features as its keys, if audio feature data is missing for the track, + an empty dictionary is returned. """ response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json() + if 'error' in response: + return {} features_dict = {} # Data that we don't need