From 5756642521119841ed19fecc752d386630584989 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 28 Jun 2018 12:28:12 -0400 Subject: [PATCH 1/9] Added complete console logging for parsing library Specifically, when making API calls for genres/features and updating genres for tracks in db. --- recreate-db.txt | 6 ++---- spotifyvis/migrations/__init__.py | 0 spotifyvis/models.py | 3 +-- spotifyvis/utils.py | 32 ++++++++++++++++++++++--------- spotifyvis/views.py | 24 ++++++++++++++++++++--- 5 files changed, 47 insertions(+), 18 deletions(-) delete mode 100644 spotifyvis/migrations/__init__.py diff --git a/recreate-db.txt b/recreate-db.txt index 5c1e574..cb082e7 100644 --- a/recreate-db.txt +++ b/recreate-db.txt @@ -1,8 +1,6 @@ -# https://stackoverflow.com/a/34576062/8811872 - -sudo su postgres -psql +sudo -u postgres psql drop database spotifyvis; create database spotifyvis with owner django; + \q exit diff --git a/spotifyvis/migrations/__init__.py b/spotifyvis/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 7419443..09a9ec7 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -28,8 +28,7 @@ class Artist(models.Model): verbose_name_plural = "Artists" artist_id = models.CharField(primary_key=True, max_length=MAX_ID) - # unique since only storing one genre per artist right now - name = models.CharField(unique=True, max_length=50) + name = models.CharField(max_length=50) genres = models.ManyToManyField(Genre, blank=True) def __str__(self): diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 668c274..dd0b087 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -11,12 +11,16 @@ import json # }}} imports # +# API limits {{{ # + USER_TRACKS_LIMIT = 50 ARTIST_LIMIT = 50 FEATURES_LIMIT = 100 # ARTIST_LIMIT = 25 # FEATURES_LIMIT = 25 +# }}} API limits # + # parse_library {{{ # def parse_library(headers, tracks, user): @@ -63,10 +67,8 @@ def parse_library(headers, tracks, user): # }}} add artists # - # TODO: fix this, don't need any more - top_genre = "" track_obj, track_created = save_track_obj(track_dict['track'], - track_artists, top_genre, user) + track_artists, user) # add audio features {{{ # @@ -80,8 +82,8 @@ def parse_library(headers, tracks, user): # }}} add audio features # - # temporary console logging - print("#{}-{}: {} - {}".format(offset + 1, + # console logging + print("Added track #{}-{}: {} - {}".format(offset + 1, offset + USER_TRACKS_LIMIT, track_obj.artists.first(), track_obj.name)) @@ -131,13 +133,15 @@ def update_track_genres(user): track.genre = most_common_genre if most_common_genre is not None \ else undefined_genre_obj track.save() - # print(track.name, track.genre) + + # console logging + print("Added '{}' as genre for song '{}'".format(track.genre, track.name)) # }}} update_track_genres # # save_track_obj {{{ # -def save_track_obj(track_dict, artists, top_genre, user): +def save_track_obj(track_dict, artists, user): """Make an entry in the database for this track if it doesn't exist already. :track_dict: dictionary from the API call containing track information. @@ -158,7 +162,6 @@ def save_track_obj(track_dict, artists, top_genre, user): popularity=int(track_dict['popularity']), runtime=int(float(track_dict['duration_ms']) / 1000), name=track_dict['name'], - # genre=top_genre, ) # have to add artists and user after saving object since track needs to @@ -200,8 +203,14 @@ def get_audio_features(headers, track_objs): setattr(cur_features_obj, key, val) cur_features_obj.save() + # console logging + print("Added features for song #{} - {}".format(i + 1, + track_objs[i].name)) + # }}} get_audio_features # +# process_artist_genre {{{ # + def process_artist_genre(genre_name, artist_obj): """Increase count for correspoding Genre object to genre_name and add that Genre to artist_obj. @@ -219,6 +228,8 @@ def process_artist_genre(genre_name, artist_obj): artist_obj.genres.add(genre_obj) artist_obj.save() +# }}} process_artist_genre # + # add_artist_genres {{{ # def add_artist_genres(headers, artist_objs): @@ -236,7 +247,6 @@ def add_artist_genres(headers, artist_objs): params = {'ids': artist_ids} artists_response = requests.get('https://api.spotify.com/v1/artists/', headers=headers, params=params).json()['artists'] - # pprint.pprint(artists_response) for i in range(len(artist_objs)): if len(artists_response[i]['genres']) == 0: process_artist_genre("undefined", artist_objs[i]) @@ -244,6 +254,10 @@ def add_artist_genres(headers, artist_objs): for genre in artists_response[i]['genres']: process_artist_genre(genre, artist_objs[i]) + # console logging + print("Added genres for artist #{} - {}".format(i + 1, + artist_objs[i].name)) + # }}} add_artist_genres # # get_artists_in_genre {{{ # diff --git a/spotifyvis/views.py b/spotifyvis/views.py index d48acef..dc2d708 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -18,9 +18,13 @@ from .models import User, Track, AudioFeatures, Artist # }}} imports # +# global vars {{{ # + TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' TRACKS_TO_QUERY = 200 +# }}} global vars # + # generate_random_string {{{ # @@ -66,7 +70,8 @@ def index(request): # uses Authorization Code flow def login(request): - # use a randomly generated state string to prevent cross-site request forgery attacks + # use a randomly generated state string to prevent cross-site request + # forgery attacks state_str = generate_random_string(16) request.session['state_string'] = state_str @@ -79,7 +84,8 @@ def login(request): 'show_dialog': False } - params = urllib.parse.urlencode(payload) # turn the payload dict into a query string + # turn the payload dict into a query string + params = urllib.parse.urlencode(payload) authorize_url = "https://accounts.spotify.com/authorize/?{}".format(params) return redirect(authorize_url) @@ -170,8 +176,10 @@ def user_data(request): # }}} user_data # +# admin_graphs {{{ # + def admin_graphs(request): - """TODO + """Redirect to logged in page as ourselves. For testing purposes. """ user_id = "polarbier" # user_id = "chrisshyi13" @@ -183,6 +191,8 @@ def admin_graphs(request): update_track_genres(user_obj) return render(request, 'spotifyvis/logged_in.html', context) +# }}} admin_graphs # + # get_artist_data {{{ # def get_artist_data(request, user_secret): @@ -197,6 +207,8 @@ def get_artist_data(request, user_secret): # }}} get_artist_data # +# display_genre_graph {{{ # + def display_genre_graph(request, client_secret): user = User.objects.get(user_secret=client_secret) context = { @@ -204,6 +216,10 @@ def display_genre_graph(request, client_secret): } return render(request, "spotifyvis/genre_graph.html", context) +# }}} display_genre_graph # + +# audio_features graph {{{ # + def audio_features(request, client_secret): user = User.objects.get(user_secret=client_secret) context = { @@ -212,6 +228,8 @@ def audio_features(request, client_secret): } return render(request, "spotifyvis/audio_features.html", context) +# }}} audio_features graph # + # get_audio_feature_data {{{ # def get_audio_feature_data(request, audio_feature, client_secret): From a0a1b861646800500636631471e63be14e28e7f5 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 28 Jun 2018 13:11:51 -0400 Subject: [PATCH 2/9] Implement getting user's entire library By passing 0 as num_tracks to parse_library. --- spotifyvis/utils.py | 19 ++++++++++--------- spotifyvis/views.py | 3 ++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index dd0b087..c459904 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -23,31 +23,32 @@ FEATURES_LIMIT = 100 # parse_library {{{ # -def parse_library(headers, tracks, user): - """Scans user's library for certain number of tracks and store the information in a database +def parse_library(headers, num_tracks, user): + """Scans user's library for num_tracks and store the information in a database :headers: For API call. - :tracks: Number of tracks to get from user's library. + :num_tracks: Number of tracks to get from user's library (0 scans the entire + library). :user: a User object representing the user whose library we are parsing :returns: None """ - # TODO: implement importing entire library with 0 as tracks param - # keeps track of point to get songs from offset = 0 payload = {'limit': str(USER_TRACKS_LIMIT)} artist_genre_queue = [] features_queue = [] - # iterate until hit requested num of tracks - for i in range(0, tracks, USER_TRACKS_LIMIT): + # create this obj so loop runs at least once + saved_tracks_response = [0] + # scan until reach num_tracks or no tracks left if scanning entire library + while (num_tracks == 0 or offset < num_tracks) and len(saved_tracks_response) > 0: payload['offset'] = str(offset) saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', headers=headers, - params=payload).json() + params=payload).json()['items'] - for track_dict in saved_tracks_response['items']: + for track_dict in saved_tracks_response: # add artists {{{ # # update artist info before track so that Track object can reference diff --git a/spotifyvis/views.py b/spotifyvis/views.py index dc2d708..ed25b54 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -21,7 +21,8 @@ from .models import User, Track, AudioFeatures, Artist # global vars {{{ # TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' -TRACKS_TO_QUERY = 200 +# TRACKS_TO_QUERY = 200 +TRACKS_TO_QUERY = 0 # }}} global vars # From 8faf229df9c729fd30ffcc0ec1649457af5081c7 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 28 Jun 2018 13:21:38 -0400 Subject: [PATCH 3/9] Added global var to toggle console logging --- spotifyvis/utils.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index c459904..067d631 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -11,7 +11,10 @@ import json # }}} imports # -# API limits {{{ # +# global vars {{{ # + +console_logging = True +# console_logging = False USER_TRACKS_LIMIT = 50 ARTIST_LIMIT = 50 @@ -19,7 +22,7 @@ FEATURES_LIMIT = 100 # ARTIST_LIMIT = 25 # FEATURES_LIMIT = 25 -# }}} API limits # +# }}} global vars # # parse_library {{{ # @@ -83,11 +86,11 @@ def parse_library(headers, num_tracks, user): # }}} add audio features # - # console logging - print("Added track #{}-{}: {} - {}".format(offset + 1, - offset + USER_TRACKS_LIMIT, - track_obj.artists.first(), - track_obj.name)) + if console_logging: + print("Added track #{}-{}: {} - {}".format(offset + 1, + offset + USER_TRACKS_LIMIT, + track_obj.artists.first(), + track_obj.name)) # calculates num_songs with offset + songs retrieved offset += USER_TRACKS_LIMIT @@ -135,8 +138,8 @@ def update_track_genres(user): else undefined_genre_obj track.save() - # console logging - print("Added '{}' as genre for song '{}'".format(track.genre, track.name)) + if console_logging: + print("Added '{}' as genre for song '{}'".format(track.genre, track.name)) # }}} update_track_genres # @@ -204,9 +207,9 @@ def get_audio_features(headers, track_objs): setattr(cur_features_obj, key, val) cur_features_obj.save() - # console logging - print("Added features for song #{} - {}".format(i + 1, - track_objs[i].name)) + if console_logging: + print("Added features for song #{} - {}".format(i + 1, + track_objs[i].name)) # }}} get_audio_features # @@ -255,9 +258,9 @@ def add_artist_genres(headers, artist_objs): for genre in artists_response[i]['genres']: process_artist_genre(genre, artist_objs[i]) - # console logging - print("Added genres for artist #{} - {}".format(i + 1, - artist_objs[i].name)) + if console_logging: + print("Added genres for artist #{} - {}".format(i + 1, + artist_objs[i].name)) # }}} add_artist_genres # From 22ea4728897e71e181c09bccd45397c650aced86 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 28 Jun 2018 14:06:58 -0400 Subject: [PATCH 4/9] Store user's total songs Also fixed "lag" to access admin charts page. --- spotifyvis/models.py | 1 + spotifyvis/utils.py | 10 ++++++---- spotifyvis/views.py | 15 ++++++++++----- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/spotifyvis/models.py b/spotifyvis/models.py index 09a9ec7..9ac0743 100644 --- a/spotifyvis/models.py +++ b/spotifyvis/models.py @@ -45,6 +45,7 @@ class User(models.Model): user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID user_secret = models.CharField(max_length=50, default='') + total_songs = models.PositiveIntegerField() def __str__(self): return self.user_id diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index 067d631..cf62e5e 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -191,9 +191,10 @@ def get_audio_features(headers, track_objs): :returns: None """ track_ids = str.join(",", [track_obj.track_id for track_obj in track_objs]) - params = {'ids': track_ids} features_response = requests.get("https://api.spotify.com/v1/audio-features", - headers=headers,params=params).json()['audio_features'] + headers=headers, + params={'ids': track_ids} + ).json()['audio_features'] # pprint.pprint(features_response) useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", ] @@ -248,9 +249,10 @@ def add_artist_genres(headers, artist_objs): """ artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs]) - params = {'ids': artist_ids} artists_response = requests.get('https://api.spotify.com/v1/artists/', - headers=headers, params=params).json()['artists'] + headers=headers, + params={'ids': artist_ids}, + ).json()['artists'] for i in range(len(artist_objs)): if len(artists_response[i]['genres']) == 0: process_artist_genre("undefined", artist_objs[i]) diff --git a/spotifyvis/views.py b/spotifyvis/views.py index ed25b54..662b6d8 100644 --- a/spotifyvis/views.py +++ b/spotifyvis/views.py @@ -21,8 +21,8 @@ from .models import User, Track, AudioFeatures, Artist # global vars {{{ # TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' -# TRACKS_TO_QUERY = 200 -TRACKS_TO_QUERY = 0 +TRACKS_TO_QUERY = 200 +# TRACKS_TO_QUERY = 0 # }}} global vars # @@ -157,13 +157,19 @@ def user_data(request): # create user obj {{{ # + total_songs = requests.get('https://api.spotify.com/v1/me/tracks', + headers=headers, + params={'limit': '1'}).json()['total'] try: user = User.objects.get(user_id=user_data_response['id']) except User.DoesNotExist: # Python docs recommends 32 bytes of randomness against brute force attacks - user = User(user_id=user_data_response['id'], user_secret=secrets.token_urlsafe(32)) + user = User.objects.create( + user_id=user_data_response['id'], + user_secret=secrets.token_urlsafe(32), + total_songs=total_songs, + ) request.session['user_secret'] = user.user_secret - user.save() # }}} create user obj # @@ -189,7 +195,6 @@ def admin_graphs(request): 'user_id': user_id, 'user_secret': user_obj.user_secret, } - update_track_genres(user_obj) return render(request, 'spotifyvis/logged_in.html', context) # }}} admin_graphs # From 3bcd7576cfeac962126f4ec8a1840a561afd7361 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 28 Jun 2018 14:39:34 -0400 Subject: [PATCH 5/9] Improved items processed count for logging Now keeps track of number of items processed across function calls. --- spotifyvis/utils.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/spotifyvis/utils.py b/spotifyvis/utils.py index cf62e5e..05a4609 100644 --- a/spotifyvis/utils.py +++ b/spotifyvis/utils.py @@ -13,15 +13,17 @@ import json # global vars {{{ # -console_logging = True -# console_logging = False - USER_TRACKS_LIMIT = 50 ARTIST_LIMIT = 50 FEATURES_LIMIT = 100 # ARTIST_LIMIT = 25 # FEATURES_LIMIT = 25 +console_logging = True +# console_logging = False +artists_genre_processed = 0 +features_processed = 0 + # }}} global vars # # parse_library {{{ # @@ -51,6 +53,9 @@ def parse_library(headers, num_tracks, user): headers=headers, params=payload).json()['items'] + if console_logging: + tracks_processed = 0 + for track_dict in saved_tracks_response: # add artists {{{ # @@ -87,10 +92,12 @@ def parse_library(headers, num_tracks, user): # }}} add audio features # if console_logging: - print("Added track #{}-{}: {} - {}".format(offset + 1, - offset + USER_TRACKS_LIMIT, + tracks_processed += 1 + print("Added track #{}: {} - {}".format( + offset + tracks_processed, track_obj.artists.first(), - track_obj.name)) + track_obj.name, + )) # calculates num_songs with offset + songs retrieved offset += USER_TRACKS_LIMIT @@ -121,6 +128,7 @@ def update_track_genres(user): :returns: None """ + tracks_processed = 0 user_tracks = Track.objects.filter(users__exact=user) for track in user_tracks: # just using this variable to save another call to db @@ -137,9 +145,14 @@ def update_track_genres(user): track.genre = most_common_genre if most_common_genre is not None \ else undefined_genre_obj track.save() + tracks_processed += 1 if console_logging: - print("Added '{}' as genre for song '{}'".format(track.genre, track.name)) + print("Added '{}' as genre for song #{} - '{}'".format( + track.genre, + tracks_processed, + track.name, + )) # }}} update_track_genres # @@ -190,6 +203,7 @@ def get_audio_features(headers, track_objs): :returns: None """ + track_ids = str.join(",", [track_obj.track_id for track_obj in track_objs]) features_response = requests.get("https://api.spotify.com/v1/audio-features", headers=headers, @@ -209,8 +223,10 @@ def get_audio_features(headers, track_objs): cur_features_obj.save() if console_logging: - print("Added features for song #{} - {}".format(i + 1, - track_objs[i].name)) + global features_processed + features_processed += 1 + print("Added features for song #{} - {}".format( + features_processed, track_objs[i].name)) # }}} get_audio_features # @@ -248,6 +264,7 @@ def add_artist_genres(headers, artist_objs): :returns: None """ + artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs]) artists_response = requests.get('https://api.spotify.com/v1/artists/', headers=headers, @@ -261,8 +278,10 @@ def add_artist_genres(headers, artist_objs): process_artist_genre(genre, artist_objs[i]) if console_logging: - print("Added genres for artist #{} - {}".format(i + 1, - artist_objs[i].name)) + global artists_genre_processed + artists_genre_processed += 1 + print("Added genres for artist #{} - {}".format( + artists_genre_processed, artist_objs[i].name)) # }}} add_artist_genres # From 8b1344d45308c50369ddd06cf1403e85c64f6c26 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Fri, 29 Jun 2018 04:15:08 -0400 Subject: [PATCH 6/9] Split spotifyvis code into different apps (#47) Server is able to start, but none of the apps are linked together yet. --- {musicvis => api}/__init__.py | 0 {spotifyvis => api}/admin.py | 0 api/apps.py | 4 + {spotifyvis => api}/models.py | 0 .../templates/api}/logged_in.html | 0 api/urls.py | 12 + {spotifyvis => api}/utils.py | 0 api/views.py | 86 ++++++ .../css/dark_bg.css | 0 {spotifyvis/migrations => graphs}/__init__.py | 0 graphs/apps.py | 4 + graphs/models.py | 104 ++++++++ .../static/graphs}/scripts/artist_graph.js | 0 .../static/graphs}/scripts/genre_graph.js | 0 .../templates/graphs}/artist_graph.html | 0 .../templates/graphs/features_graph.html | 0 .../templates/graphs}/genre_graph.html | 0 graphs/urls.py | 12 + graphs/views.py | 51 ++++ login/__init__.py | 0 login/apps.py | 4 + .../templates/login}/index.html | 0 login/urls.py | 11 + {spotifyvis => login}/views.py | 115 +------- manage.py | 2 +- musicvis/urls.py | 22 -- sample-track-obj.py | 250 ------------------ spotifyvis/apps.py | 5 - {musicvis => spotifyvis}/settings.py | 8 +- spotifyvis/static/spotifyvis/scripts/index.js | 42 --- .../templates/spotifyvis/user_data.html | 21 -- spotifyvis/tests.py | 67 ----- spotifyvis/urls.py | 35 +-- {musicvis => spotifyvis}/wsgi.py | 2 +- 34 files changed, 319 insertions(+), 538 deletions(-) rename {musicvis => api}/__init__.py (100%) rename {spotifyvis => api}/admin.py (100%) create mode 100644 api/apps.py rename {spotifyvis => api}/models.py (100%) rename {spotifyvis/templates/spotifyvis => api/templates/api}/logged_in.html (100%) create mode 100644 api/urls.py rename {spotifyvis => api}/utils.py (100%) create mode 100644 api/views.py rename {spotifyvis/static/spotifyvis => common-static}/css/dark_bg.css (100%) rename {spotifyvis/migrations => graphs}/__init__.py (100%) create mode 100644 graphs/apps.py create mode 100644 graphs/models.py rename {spotifyvis/static/spotifyvis => graphs/static/graphs}/scripts/artist_graph.js (100%) rename {spotifyvis/static/spotifyvis => graphs/static/graphs}/scripts/genre_graph.js (100%) rename {spotifyvis/templates/spotifyvis => graphs/templates/graphs}/artist_graph.html (100%) rename spotifyvis/templates/spotifyvis/audio_features.html => graphs/templates/graphs/features_graph.html (100%) rename {spotifyvis/templates/spotifyvis => graphs/templates/graphs}/genre_graph.html (100%) create mode 100644 graphs/urls.py create mode 100644 graphs/views.py create mode 100644 login/__init__.py create mode 100644 login/apps.py rename {spotifyvis/templates/spotifyvis => login/templates/login}/index.html (100%) create mode 100644 login/urls.py rename {spotifyvis => login}/views.py (58%) delete mode 100644 musicvis/urls.py delete mode 100644 sample-track-obj.py delete mode 100644 spotifyvis/apps.py rename {musicvis => spotifyvis}/settings.py (94%) delete mode 100644 spotifyvis/static/spotifyvis/scripts/index.js delete mode 100644 spotifyvis/templates/spotifyvis/user_data.html delete mode 100644 spotifyvis/tests.py rename {musicvis => spotifyvis}/wsgi.py (82%) diff --git a/musicvis/__init__.py b/api/__init__.py similarity index 100% rename from musicvis/__init__.py rename to api/__init__.py diff --git a/spotifyvis/admin.py b/api/admin.py similarity index 100% rename from spotifyvis/admin.py rename to api/admin.py diff --git a/api/apps.py b/api/apps.py new file mode 100644 index 0000000..c147de3 --- /dev/null +++ b/api/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class ApiConfig(AppConfig): + name = 'api' diff --git a/spotifyvis/models.py b/api/models.py similarity index 100% rename from spotifyvis/models.py rename to api/models.py diff --git a/spotifyvis/templates/spotifyvis/logged_in.html b/api/templates/api/logged_in.html similarity index 100% rename from spotifyvis/templates/spotifyvis/logged_in.html rename to api/templates/api/logged_in.html diff --git a/api/urls.py b/api/urls.py new file mode 100644 index 0000000..204f9c8 --- /dev/null +++ b/api/urls.py @@ -0,0 +1,12 @@ +from django.urls import path, include + +from .views import * + +urlpatterns = [ + path('user_artists/', get_artist_data, + name='get_artist_data'), + path('user_genres/', get_genre_data, + name='get_genre_data'), + path('audio_features//', + get_audio_feature_data, name='get_audio_feature_data'), +] diff --git a/spotifyvis/utils.py b/api/utils.py similarity index 100% rename from spotifyvis/utils.py rename to api/utils.py diff --git a/api/views.py b/api/views.py new file mode 100644 index 0000000..a3dfc27 --- /dev/null +++ b/api/views.py @@ -0,0 +1,86 @@ +# imports {{{ # + +import math +import random +import requests +import os +import urllib +import secrets +import pprint +import string +from datetime import datetime + +from django.http import JsonResponse +from django.db.models import Count, Q +from .utils import parse_library, get_artists_in_genre, update_track_genres +from .models import User, Track, AudioFeatures, Artist + +# }}} imports # + +TRACKS_TO_QUERY = 200 + +# get_artist_data {{{ # + + +def get_artist_data(request, user_secret): + """Returns artist data as a JSON serialized list of dictionaries + The (key, value) pairs are (artist name, song count for said artist) + + :param request: the HTTP request + :param user_secret: the user secret used for identification + :return: a JsonResponse + """ + user = User.objects.get(user_secret=user_secret) + artist_counts = Artist.objects.annotate(num_songs=Count('track', + filter=Q(track__users=user))) + processed_artist_counts = [{'name': artist.name, + 'num_songs': artist.num_songs} for artist in artist_counts] + return JsonResponse(data=processed_artist_counts, safe=False) + +# }}} get_artist_data # + +# get_audio_feature_data {{{ # + +def get_audio_feature_data(request, audio_feature, user_secret): + """Returns all data points for a given audio feature + + Args: + request: the HTTP request + audio_feature: The audio feature to be queried + user_secret: client secret, used to identify the user + """ + user = User.objects.get(user_secret=user_secret) + user_tracks = Track.objects.filter(users=user) + response_payload = { + 'data_points': [], + } + for track in user_tracks: + try: + audio_feature_obj = AudioFeatures.objects.get(track=track) + response_payload['data_points'].append(getattr(audio_feature_obj, audio_feature)) + except AudioFeatures.DoesNotExist: + continue + return JsonResponse(response_payload) + +# }}} get_audio_feature_data # + +# get_genre_data {{{ # + +def get_genre_data(request, user_secret): + """Return genre data needed to create the graph user. + TODO + """ + user = User.objects.get(user_secret=user_secret) + genre_counts = (Track.objects.filter(users__exact=user) + .values('genre') + .order_by('genre') + .annotate(num_songs=Count('genre')) + ) + for genre_dict in genre_counts: + genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre'], + genre_dict['num_songs']) + print("*** Genre Breakdown ***") + pprint.pprint(list(genre_counts)) + return JsonResponse(data=list(genre_counts), safe=False) + +# }}} get_genre_data # diff --git a/spotifyvis/static/spotifyvis/css/dark_bg.css b/common-static/css/dark_bg.css similarity index 100% rename from spotifyvis/static/spotifyvis/css/dark_bg.css rename to common-static/css/dark_bg.css diff --git a/spotifyvis/migrations/__init__.py b/graphs/__init__.py similarity index 100% rename from spotifyvis/migrations/__init__.py rename to graphs/__init__.py diff --git a/graphs/apps.py b/graphs/apps.py new file mode 100644 index 0000000..6b4b056 --- /dev/null +++ b/graphs/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class GraphsConfig(AppConfig): + name = 'graphs' diff --git a/graphs/models.py b/graphs/models.py new file mode 100644 index 0000000..7419443 --- /dev/null +++ b/graphs/models.py @@ -0,0 +1,104 @@ +from django.db import models + +# id's are 22 in length in examples but set to 30 for buffer +MAX_ID = 30 + +# Genre {{{ # + +class Genre(models.Model): + + class Meta: + verbose_name = "Genre" + verbose_name_plural = "Genres" + + name = models.CharField(primary_key=True, max_length=50) + num_songs = models.PositiveIntegerField() + + def __str__(self): + return self.name + +# }}} Genre # + +# Artist {{{ # + + +class Artist(models.Model): + class Meta: + verbose_name = "Artist" + verbose_name_plural = "Artists" + + artist_id = models.CharField(primary_key=True, max_length=MAX_ID) + # unique since only storing one genre per artist right now + name = models.CharField(unique=True, max_length=50) + genres = models.ManyToManyField(Genre, blank=True) + + def __str__(self): + return self.name + +# }}} Artist # + +# User {{{ # + +class User(models.Model): + class Meta: + verbose_name = "User" + verbose_name_plural = "Users" + + user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID + user_secret = models.CharField(max_length=50, default='') + + def __str__(self): + return self.user_id + +# }}} User # + +# Track {{{ # + +class Track(models.Model): + + class Meta: + verbose_name = "Track" + verbose_name_plural = "Tracks" + + track_id = models.CharField(primary_key=True, max_length=MAX_ID) + # artist = models.ForeignKey(Artist, on_delete=models.CASCADE) + artists = models.ManyToManyField(Artist, blank=True) + year = models.PositiveSmallIntegerField() + popularity = models.PositiveSmallIntegerField() + runtime = models.PositiveSmallIntegerField() + name = models.CharField(max_length=200) + users = models.ManyToManyField(User, blank=True) + genre = models.ForeignKey(Genre, on_delete=models.CASCADE, blank=True, + null=True) + + def __str__(self): + track_str = "{}, genre: {}, artists: [".format(self.name, self.genre) + for artist in self.artists.all(): + track_str += "{}, ".format(artist.name) + track_str += "]" + return track_str + +# }}} Track # + +# AudioFeatures {{{ # + +class AudioFeatures(models.Model): + + class Meta: + verbose_name = "AudioFeatures" + verbose_name_plural = "AudioFeatures" + + track = models.OneToOneField(Track, on_delete=models.CASCADE, primary_key=True,) + acousticness = models.DecimalField(decimal_places=3, max_digits=3) + danceability = models.DecimalField(decimal_places=3, max_digits=3) + energy = models.DecimalField(decimal_places=3, max_digits=3) + instrumentalness = models.DecimalField(decimal_places=3, max_digits=3) + loudness = models.DecimalField(decimal_places=3, max_digits=6) + speechiness = models.DecimalField(decimal_places=3, max_digits=3) + tempo = models.DecimalField(decimal_places=3, max_digits=6) + valence = models.DecimalField(decimal_places=3, max_digits=3) + + def __str__(self): + return super(AudioFeatures, self).__str__() + +# }}} AudioFeatures # diff --git a/spotifyvis/static/spotifyvis/scripts/artist_graph.js b/graphs/static/graphs/scripts/artist_graph.js similarity index 100% rename from spotifyvis/static/spotifyvis/scripts/artist_graph.js rename to graphs/static/graphs/scripts/artist_graph.js diff --git a/spotifyvis/static/spotifyvis/scripts/genre_graph.js b/graphs/static/graphs/scripts/genre_graph.js similarity index 100% rename from spotifyvis/static/spotifyvis/scripts/genre_graph.js rename to graphs/static/graphs/scripts/genre_graph.js diff --git a/spotifyvis/templates/spotifyvis/artist_graph.html b/graphs/templates/graphs/artist_graph.html similarity index 100% rename from spotifyvis/templates/spotifyvis/artist_graph.html rename to graphs/templates/graphs/artist_graph.html diff --git a/spotifyvis/templates/spotifyvis/audio_features.html b/graphs/templates/graphs/features_graph.html similarity index 100% rename from spotifyvis/templates/spotifyvis/audio_features.html rename to graphs/templates/graphs/features_graph.html diff --git a/spotifyvis/templates/spotifyvis/genre_graph.html b/graphs/templates/graphs/genre_graph.html similarity index 100% rename from spotifyvis/templates/spotifyvis/genre_graph.html rename to graphs/templates/graphs/genre_graph.html diff --git a/graphs/urls.py b/graphs/urls.py new file mode 100644 index 0000000..9bfcf99 --- /dev/null +++ b/graphs/urls.py @@ -0,0 +1,12 @@ +from django.urls import path, include + +from .views import * + +urlpatterns = [ + path('artists/', artist_data, + name='display_artist_graph'), + path('genre/', display_genre_graph, + name='display_genre_graph'), + path('audio_features/', audio_features, + name='display_audio_features'), +] diff --git a/graphs/views.py b/graphs/views.py new file mode 100644 index 0000000..e65f73b --- /dev/null +++ b/graphs/views.py @@ -0,0 +1,51 @@ +# imports {{{ # + +import math +import random +import requests +import os +import urllib +import secrets +import pprint +import string +from datetime import datetime + +from django.shortcuts import render, redirect + +# }}} imports # + +def artist_data(request, user_secret): + """Renders the artist data graph display page + + :param request: the HTTP request + :param user_secret: the user secret used for identification + :return: render the artist data graph display page + """ + user = User.objects.get(user_secret=user_secret) + context = { + 'user_id': user.user_id, + 'user_secret': user_secret, + } + return render(request, "spotifyvis/artist_graph.html", context) + +def display_genre_graph(request, user_secret): + user = User.objects.get(user_secret=user_secret) + context = { + 'user_secret': user_secret, + } + return render(request, "spotifyvis/genre_graph.html", context) + + +def audio_features(request, user_secret): + """Renders the audio features page + + :param request: the HTTP request + :param user_secret: user secret used for identification + :return: renders the audio features page + """ + user = User.objects.get(user_secret=user_secret) + context = { + 'user_id': user.user_id, + 'user_secret': user_secret, + } + return render(request, "spotifyvis/audio_features.html", context) diff --git a/login/__init__.py b/login/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/login/apps.py b/login/apps.py new file mode 100644 index 0000000..645de5f --- /dev/null +++ b/login/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class LoginConfig(AppConfig): + name = 'login' diff --git a/spotifyvis/templates/spotifyvis/index.html b/login/templates/login/index.html similarity index 100% rename from spotifyvis/templates/spotifyvis/index.html rename to login/templates/login/index.html diff --git a/login/urls.py b/login/urls.py new file mode 100644 index 0000000..02ca78d --- /dev/null +++ b/login/urls.py @@ -0,0 +1,11 @@ +from django.urls import path, include + +from .views import * + +urlpatterns = [ + path('', index, name='index'), + path('spotify_login', spotify_login, name='spotify_login'), + path('callback', callback, name='callback'), + path('user_data', user_data, name='user_data'), + path('admin_graphs', admin_graphs, name='admin_graphs'), +] diff --git a/spotifyvis/views.py b/login/views.py similarity index 58% rename from spotifyvis/views.py rename to login/views.py index ef9b391..c479a19 100644 --- a/spotifyvis/views.py +++ b/login/views.py @@ -11,10 +11,7 @@ import string from datetime import datetime from django.shortcuts import render, redirect -from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse -from django.db.models import Count, Q -from .utils import parse_library, get_artists_in_genre, update_track_genres -from .models import User, Track, AudioFeatures, Artist +from django.http import HttpResponseBadRequest # }}} imports # @@ -65,7 +62,7 @@ def index(request): # login {{{ # # uses Authorization Code flow -def login(request): +def spotify_login(request): # use a randomly generated state string to prevent cross-site request forgery attacks state_str = generate_random_string(16) request.session['state_string'] = state_str @@ -117,7 +114,6 @@ def callback(request): # user_data {{{ # - def user_data(request): # get user token {{{ # @@ -165,7 +161,8 @@ def user_data(request): 'user_secret': user.user_secret, } - parse_library(headers, TRACKS_TO_QUERY, user) + # TODO: redirect to API app to parse library or loading page + # parse_library(headers, TRACKS_TO_QUERY, user) return render(request, 'spotifyvis/logged_in.html', context) # }}} user_data # @@ -182,107 +179,3 @@ def admin_graphs(request): } update_track_genres(user_obj) return render(request, 'spotifyvis/logged_in.html', context) - - -def artist_data(request, user_secret): - """Renders the artist data graph display page - - :param request: the HTTP request - :param user_secret: the user secret used for identification - :return: render the artist data graph display page - """ - user = User.objects.get(user_secret=user_secret) - context = { - 'user_id': user.user_id, - 'user_secret': user_secret, - } - return render(request, "spotifyvis/artist_graph.html", context) - -# get_artist_data {{{ # - - -def get_artist_data(request, user_secret): - """Returns artist data as a JSON serialized list of dictionaries - The (key, value) pairs are (artist name, song count for said artist) - - :param request: the HTTP request - :param user_secret: the user secret used for identification - :return: a JsonResponse - """ - user = User.objects.get(user_secret=user_secret) - artist_counts = Artist.objects.annotate(num_songs=Count('track', - filter=Q(track__users=user))) - processed_artist_counts = [{'name': artist.name, - 'num_songs': artist.num_songs} for artist in artist_counts] - return JsonResponse(data=processed_artist_counts, safe=False) - -# }}} get_artist_data # - - -def display_genre_graph(request, user_secret): - user = User.objects.get(user_secret=user_secret) - context = { - 'user_secret': user_secret, - } - return render(request, "spotifyvis/genre_graph.html", context) - - -def audio_features(request, user_secret): - """Renders the audio features page - - :param request: the HTTP request - :param user_secret: user secret used for identification - :return: renders the audio features page - """ - user = User.objects.get(user_secret=user_secret) - context = { - 'user_id': user.user_id, - 'user_secret': user_secret, - } - return render(request, "spotifyvis/audio_features.html", context) - -# get_audio_feature_data {{{ # - -def get_audio_feature_data(request, audio_feature, user_secret): - """Returns all data points for a given audio feature - - Args: - request: the HTTP request - audio_feature: The audio feature to be queried - user_secret: client secret, used to identify the user - """ - user = User.objects.get(user_secret=user_secret) - user_tracks = Track.objects.filter(users=user) - response_payload = { - 'data_points': [], - } - for track in user_tracks: - try: - audio_feature_obj = AudioFeatures.objects.get(track=track) - response_payload['data_points'].append(getattr(audio_feature_obj, audio_feature)) - except AudioFeatures.DoesNotExist: - continue - return JsonResponse(response_payload) - -# }}} get_audio_feature_data # - -# get_genre_data {{{ # - -def get_genre_data(request, user_secret): - """Return genre data needed to create the graph user. - TODO - """ - user = User.objects.get(user_secret=user_secret) - genre_counts = (Track.objects.filter(users__exact=user) - .values('genre') - .order_by('genre') - .annotate(num_songs=Count('genre')) - ) - for genre_dict in genre_counts: - genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre'], - genre_dict['num_songs']) - print("*** Genre Breakdown ***") - pprint.pprint(list(genre_counts)) - return JsonResponse(data=list(genre_counts), safe=False) - -# }}} get_genre_data # diff --git a/manage.py b/manage.py index 7162f7a..f2b8f0f 100755 --- a/manage.py +++ b/manage.py @@ -3,7 +3,7 @@ import os import sys if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "musicvis.settings") + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "spotifyvis.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/musicvis/urls.py b/musicvis/urls.py deleted file mode 100644 index c579fd9..0000000 --- a/musicvis/urls.py +++ /dev/null @@ -1,22 +0,0 @@ -"""musicdata URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/2.0/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: path('', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.urls import include, path - 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) -""" -from django.contrib import admin -from django.urls import path, include - -urlpatterns = [ - path('', include('spotifyvis.urls')), - path('admin/', admin.site.urls), -] diff --git a/sample-track-obj.py b/sample-track-obj.py deleted file mode 100644 index c3a24ed..0000000 --- a/sample-track-obj.py +++ /dev/null @@ -1,250 +0,0 @@ -{ - 'added_at':'2018-05-18T19:16:36Z', - 'track':{ - 'album':{ - 'album_type':'single', - 'artists':[ - { - 'external_urls':{ - 'spotify':'https://open.spotify.com/artist/64KEffDW9EtZ1y2vBYgq8T' - }, - 'href':'https://api.spotify.com/v1/artists/64KEffDW9EtZ1y2vBYgq8T', - 'id':'64KEffDW9EtZ1y2vBYgq8T', - 'name':'Marshmello', - 'type':'artist', - 'uri':'spotify:artist:64KEffDW9EtZ1y2vBYgq8T' - }, - { - 'external_urls':{ - 'spotify':'https://open.spotify.com/artist/5gCRApTajqwbnHHPbr2Fpi' - }, - 'href':'https://api.spotify.com/v1/artists/5gCRApTajqwbnHHPbr2Fpi', - 'id':'5gCRApTajqwbnHHPbr2Fpi', - 'name':'Juicy J', - 'type':'artist', - 'uri':'spotify:artist:5gCRApTajqwbnHHPbr2Fpi' - }, - { - 'external_urls':{ - 'spotify':'https://open.spotify.com/artist/4IWBUUAFIplrNtaOHcJPRM' - }, - 'href':'https://api.spotify.com/v1/artists/4IWBUUAFIplrNtaOHcJPRM', - 'id':'4IWBUUAFIplrNtaOHcJPRM', - 'name':'James Arthur', - 'type':'artist', - 'uri':'spotify:artist:4IWBUUAFIplrNtaOHcJPRM' - } - ], - 'available_markets':[ - 'AD', - 'AR', - 'AT', - 'AU', - 'BE', - 'BG', - 'BO', - 'BR', - 'CA', - 'CH', - 'CL', - 'CO', - 'CR', - 'CY', - 'CZ', - 'DE', - 'DK', - 'DO', - 'EC', - 'EE', - 'ES', - 'FI', - 'FR', - 'GB', - 'GR', - 'GT', - 'HK', - 'HN', - 'HU', - 'ID', - 'IE', - 'IL', - 'IS', - 'IT', - 'JP', - 'LI', - 'LT', - 'LU', - 'LV', - 'MC', - 'MT', - 'MX', - 'MY', - 'NI', - 'NL', - 'NO', - 'NZ', - 'PA', - 'PE', - 'PH', - 'PL', - 'PT', - 'PY', - 'RO', - 'SE', - 'SG', - 'SK', - 'SV', - 'TH', - 'TR', - 'TW', - 'US', - 'UY', - 'VN', - 'ZA' - ], - 'external_urls':{ - 'spotify':'https://open.spotify.com/album/6TvqOieExu0IJb9Q1gOoCz' - }, - 'href':'https://api.spotify.com/v1/albums/6TvqOieExu0IJb9Q1gOoCz', - 'id':'6TvqOieExu0IJb9Q1gOoCz', - 'images':[ - { - 'height':640, - 'url':'https://i.scdn.co/image/b3556956b8e4881c85228ada91aa953e5c0458ef', - 'width':640 - }, - { - 'height':300, - 'url':'https://i.scdn.co/image/d76072f5ca739466bd27f42f3356fa1a38c6a92d', - 'width':300 - }, - { - 'height':64, - 'url':'https://i.scdn.co/image/bfd092dfa503566d9c9a3042f213fe02bed8a5cc', - 'width':64 - } - ], - 'name':'You Can Cry', - 'release_date':'2018-05-04', - 'release_date_precision':'day', - 'type':'album', - 'uri':'spotify:album:6TvqOieExu0IJb9Q1gOoCz' - }, - 'artists':[ - { - 'external_urls':{ - 'spotify':'https://open.spotify.com/artist/64KEffDW9EtZ1y2vBYgq8T' - }, - 'href':'https://api.spotify.com/v1/artists/64KEffDW9EtZ1y2vBYgq8T', - 'id':'64KEffDW9EtZ1y2vBYgq8T', - 'name':'Marshmello', - 'type':'artist', - 'uri':'spotify:artist:64KEffDW9EtZ1y2vBYgq8T' - }, - { - 'external_urls':{ - 'spotify':'https://open.spotify.com/artist/5gCRApTajqwbnHHPbr2Fpi' - }, - 'href':'https://api.spotify.com/v1/artists/5gCRApTajqwbnHHPbr2Fpi', - 'id':'5gCRApTajqwbnHHPbr2Fpi', - 'name':'Juicy J', - 'type':'artist', - 'uri':'spotify:artist:5gCRApTajqwbnHHPbr2Fpi' - }, - { - 'external_urls':{ - 'spotify':'https://open.spotify.com/artist/4IWBUUAFIplrNtaOHcJPRM' - }, - 'href':'https://api.spotify.com/v1/artists/4IWBUUAFIplrNtaOHcJPRM', - 'id':'4IWBUUAFIplrNtaOHcJPRM', - 'name':'James Arthur', - 'type':'artist', - 'uri':'spotify:artist:4IWBUUAFIplrNtaOHcJPRM' - } - ], - 'available_markets':[ - 'AD', - 'AR', - 'AT', - 'AU', - 'BE', - 'BG', - 'BO', - 'BR', - 'CA', - 'CH', - 'CL', - 'CO', - 'CR', - 'CY', - 'CZ', - 'DE', - 'DK', - 'DO', - 'EC', - 'EE', - 'ES', - 'FI', - 'FR', - 'GB', - 'GR', - 'GT', - 'HK', - 'HN', - 'HU', - 'ID', - 'IE', - 'IL', - 'IS', - 'IT', - 'JP', - 'LI', - 'LT', - 'LU', - 'LV', - 'MC', - 'MT', - 'MX', - 'MY', - 'NI', - 'NL', - 'NO', - 'NZ', - 'PA', - 'PE', - 'PH', - 'PL', - 'PT', - 'PY', - 'RO', - 'SE', - 'SG', - 'SK', - 'SV', - 'TH', - 'TR', - 'TW', - 'US', - 'UY', - 'VN', - 'ZA' - ], - 'disc_number':1, - 'duration_ms':194533, - 'explicit':False, - 'external_ids':{ - 'isrc':'USQX91800946' - }, - 'external_urls':{ - 'spotify':'https://open.spotify.com/track/3ZbJMlEL4Kcme0ONRO7Slx' - }, - 'href':'https://api.spotify.com/v1/tracks/3ZbJMlEL4Kcme0ONRO7Slx', - 'id':'3ZbJMlEL4Kcme0ONRO7Slx', - 'name':'You Can Cry', - 'popularity':81, - 'preview_url':'https://p.scdn.co/mp3-preview/6c31f3dee18a1e7c452ce9b6948a6e04aa7629d6?cid=aefd4e45060d4f9ba5bea0f6e6d36359', - 'track_number':1, - 'type':'track', - 'uri':'spotify:track:3ZbJMlEL4Kcme0ONRO7Slx' - } -} diff --git a/spotifyvis/apps.py b/spotifyvis/apps.py deleted file mode 100644 index a232292..0000000 --- a/spotifyvis/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class SpotifyvisConfig(AppConfig): - name = 'spotifyvis' diff --git a/musicvis/settings.py b/spotifyvis/settings.py similarity index 94% rename from musicvis/settings.py rename to spotifyvis/settings.py index 0cedb25..7d951f5 100644 --- a/musicvis/settings.py +++ b/spotifyvis/settings.py @@ -37,7 +37,9 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'spotifyvis.apps.SpotifyvisConfig', + 'login.apps.LoginConfig', + 'api.apps.ApiConfig', + 'graphs.apps.GraphsConfig', ] MIDDLEWARE = [ @@ -50,7 +52,7 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -ROOT_URLCONF = 'musicvis.urls' +ROOT_URLCONF = 'spotifyvis.urls' TEMPLATES = [ { @@ -68,7 +70,7 @@ TEMPLATES = [ }, ] -WSGI_APPLICATION = 'musicvis.wsgi.application' +WSGI_APPLICATION = 'spotifyvis.wsgi.application' # Database diff --git a/spotifyvis/static/spotifyvis/scripts/index.js b/spotifyvis/static/spotifyvis/scripts/index.js deleted file mode 100644 index cfac1d1..0000000 --- a/spotifyvis/static/spotifyvis/scripts/index.js +++ /dev/null @@ -1,42 +0,0 @@ -document.getElementById("login-btn").addEventListener("click", function() { - let httpRequest = new XMLHttpRequest(); - - /* - * Handler for the response - */ - httpRequest.onreadystatechange = function() { - if (httpRequest.readyState === XMLHttpRequest.DONE) { - if (httpRequest.status === 200) { - // hide the login button - document.getElementById('login').setAttribute("display", "none"); - - let responseData = JSON.parse(httpRequest.responseText); - let dataList = document.getElementById("data-list"); - - - for (let key in responseData) { - let newLi = document.createElement("li"); - let innerList = document.createElement("ul"); - - let dataLabel = document.createElement("li"); - dataLabel.innerText = key; - - let dataValue = document.createElement("li"); - dataValue.innerText = responseData[key]; - - innerList.appendChild(dataLabel); - innerList.appendChild(dataValue); - - newLi.appendChild(innerList); - dataList.appendChild(newLi); - } - } else { - alert("There was a problem with the login request, please try again!"); - } - } - } - - httpRequest.open('GET', '/login', true); - httpRequest.send(); -}); - diff --git a/spotifyvis/templates/spotifyvis/user_data.html b/spotifyvis/templates/spotifyvis/user_data.html deleted file mode 100644 index 67c99db..0000000 --- a/spotifyvis/templates/spotifyvis/user_data.html +++ /dev/null @@ -1,21 +0,0 @@ -{% load static %} - - - - - - - - - User Spotify Data - - - - - - -

Logged in as {{ id }}

- - diff --git a/spotifyvis/tests.py b/spotifyvis/tests.py deleted file mode 100644 index ed439b4..0000000 --- a/spotifyvis/tests.py +++ /dev/null @@ -1,67 +0,0 @@ -from django.test import TestCase -from .utils import update_std_dev -import math -# Create your tests here. - -class UpdateStdDevTest(TestCase): - - def test_two_data_points(self): - """ - tests if update_std_dev behaves correctly for two data points - """ - cur_mean = 5 - cur_std_dev = 0 - - new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 10, 2) - - self.assertTrue(math.isclose(new_mean, 7.5, rel_tol=0.01)) - self.assertTrue(math.isclose(new_std_dev, 3.5355, rel_tol=0.01)) - - - def test_three_data_points(self): - """ - tests if update_std_dev behaves correctly for three data points - """ - cur_mean = 7.5 - cur_std_dev = 3.5355 - - new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 15, 3) - - self.assertTrue(math.isclose(new_mean, 10, rel_tol=0.01)) - self.assertTrue(math.isclose(new_std_dev, 5, rel_tol=0.01)) - - - def test_four_data_points(self): - """ - tests if update_std_dev behaves correctly for four data points - """ - cur_mean = 10 - cur_std_dev = 5 - - new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 20, 4) - self.assertTrue(math.isclose(new_mean, 12.5, rel_tol=0.01)) - self.assertTrue(math.isclose(new_std_dev, 6.455, rel_tol=0.01)) - - - def test_five_data_points(self): - """ - tests if update_std_dev behaves correctly for five data points - """ - cur_mean = 12.5 - cur_std_dev = 6.455 - - new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 63, 5) - self.assertTrue(math.isclose(new_mean, 22.6, rel_tol=0.01)) - self.assertTrue(math.isclose(new_std_dev, 23.2658, rel_tol=0.01)) - - - def test_sixteen_data_points(self): - """ - tests if update_std_dev behaves correctly for sixteen data points - """ - cur_mean = 0.4441 - cur_std_dev = 0.2855 - - new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, 0.7361, 16) - self.assertTrue(math.isclose(new_mean, 0.4624, rel_tol=0.01)) - self.assertTrue(math.isclose(new_std_dev, 0.2853, rel_tol=0.01)) \ No newline at end of file diff --git a/spotifyvis/urls.py b/spotifyvis/urls.py index c28cfcd..62baf8a 100644 --- a/spotifyvis/urls.py +++ b/spotifyvis/urls.py @@ -1,19 +1,24 @@ -from django.urls import path, include +"""musicdata URL Configuration -from .views import * +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/2.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path, include urlpatterns = [ - path('', index, name='index'), - path('login', login, name='login'), - path('callback', callback, name='callback'), - path('user_data', user_data, name='user_data'), - path('admin_graphs', admin_graphs, name='admin_graphs'), - path('api/user_artists/', get_artist_data, name='get_artist_data'), - path('artists/', artist_data, name='display_artist_graph'), - path('api/user_genres/', get_genre_data, name='get_genre_data'), - path('graphs/genre/', display_genre_graph, - name='display_genre_graph'), - path('audio_features/', audio_features, name='display_audio_features'), - path('api/audio_features//', - get_audio_feature_data, name='get_audio_feature_data'), + path('admin/', admin.site.urls), + path('login/', include('login.urls')), + path('api/', include('api.urls')), + path('graphs/', include('graphs.urls')), ] diff --git a/musicvis/wsgi.py b/spotifyvis/wsgi.py similarity index 82% rename from musicvis/wsgi.py rename to spotifyvis/wsgi.py index 089d0ea..5fdd0f1 100644 --- a/musicvis/wsgi.py +++ b/spotifyvis/wsgi.py @@ -11,6 +11,6 @@ import os from django.core.wsgi import get_wsgi_application -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "musicvis.settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "spotifyvis.settings") application = get_wsgi_application() From e50d3d8476d0efb1e363291662f02f232910edfc Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Fri, 29 Jun 2018 08:43:03 -0400 Subject: [PATCH 7/9] Login app working (#47) Can login and create User object. Also now store user token info in db (closes #48). --- api/models.py | 16 +-- api/urls.py | 2 + api/utils.py | 87 -------------- api/views.py | 89 ++++++++++++++- graphs/urls.py | 1 + login/models.py | 22 ++++ login/templates/login/index.html | 9 +- login/templates/login/scan.html | 22 ++++ login/urls.py | 3 +- login/views.py | 132 +++++++++++----------- recreate-db.txt | 8 -- reset_db.sh | 18 +-- spotifyvis/settings.py | 3 + spotifyvis/urls.py | 6 +- {common-static => static}/css/dark_bg.css | 0 15 files changed, 224 insertions(+), 194 deletions(-) create mode 100644 login/models.py create mode 100644 login/templates/login/scan.html delete mode 100644 recreate-db.txt mode change 100644 => 100755 reset_db.sh rename {common-static => static}/css/dark_bg.css (100%) diff --git a/api/models.py b/api/models.py index 7419443..5303876 100644 --- a/api/models.py +++ b/api/models.py @@ -1,4 +1,5 @@ from django.db import models +from login.models import User # id's are 22 in length in examples but set to 30 for buffer MAX_ID = 30 @@ -37,21 +38,6 @@ class Artist(models.Model): # }}} Artist # -# User {{{ # - -class User(models.Model): - class Meta: - verbose_name = "User" - verbose_name_plural = "Users" - - user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID - user_secret = models.CharField(max_length=50, default='') - - def __str__(self): - return self.user_id - -# }}} User # - # Track {{{ # class Track(models.Model): diff --git a/api/urls.py b/api/urls.py index 204f9c8..1196097 100644 --- a/api/urls.py +++ b/api/urls.py @@ -2,7 +2,9 @@ from django.urls import path, include from .views import * +app_name = 'api' urlpatterns = [ + # path('scan/', get_artist_data), path('user_artists/', get_artist_data, name='get_artist_data'), path('user_genres/', get_genre_data, diff --git a/api/utils.py b/api/utils.py index 668c274..a3d1b22 100644 --- a/api/utils.py +++ b/api/utils.py @@ -17,93 +17,6 @@ FEATURES_LIMIT = 100 # ARTIST_LIMIT = 25 # FEATURES_LIMIT = 25 -# parse_library {{{ # - -def parse_library(headers, tracks, user): - """Scans user's library for certain number of tracks and store the information in a database - - :headers: For API call. - :tracks: Number of tracks to get from user's library. - :user: a User object representing the user whose library we are parsing - - :returns: None - - """ - # TODO: implement importing entire library with 0 as tracks param - # keeps track of point to get songs from - offset = 0 - payload = {'limit': str(USER_TRACKS_LIMIT)} - artist_genre_queue = [] - features_queue = [] - - # iterate until hit requested num of tracks - for i in range(0, tracks, USER_TRACKS_LIMIT): - payload['offset'] = str(offset) - saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', - headers=headers, - params=payload).json() - - for track_dict in saved_tracks_response['items']: - # add artists {{{ # - - # update artist info before track so that Track object can reference - # Artist object - track_artists = [] - for artist_dict in track_dict['track']['artists']: - artist_obj, artist_created = Artist.objects.get_or_create( - artist_id=artist_dict['id'], - name=artist_dict['name'],) - # only add/tally up artist genres if new - if artist_created: - artist_genre_queue.append(artist_obj) - if len(artist_genre_queue) == ARTIST_LIMIT: - add_artist_genres(headers, artist_genre_queue) - artist_genre_queue = [] - track_artists.append(artist_obj) - - # }}} add artists # - - # TODO: fix this, don't need any more - top_genre = "" - track_obj, track_created = save_track_obj(track_dict['track'], - track_artists, top_genre, user) - - # add audio features {{{ # - - # if a new track is not created, the associated audio feature does - # not need to be created again - if track_created: - features_queue.append(track_obj) - if len(features_queue) == FEATURES_LIMIT: - get_audio_features(headers, features_queue) - features_queue = [] - - # }}} add audio features # - - # temporary console logging - print("#{}-{}: {} - {}".format(offset + 1, - offset + USER_TRACKS_LIMIT, - track_obj.artists.first(), - track_obj.name)) - - # calculates num_songs with offset + songs retrieved - offset += USER_TRACKS_LIMIT - - # clean-up {{{ # - - # update remaining artists without genres and songs without features if - # there are any - if len(artist_genre_queue) > 0: - add_artist_genres(headers, artist_genre_queue) - if len(features_queue) > 0: - get_audio_features(headers, features_queue) - - # }}} clean-up # - - update_track_genres(user) - -# }}} parse_library # - # update_track_genres {{{ # def update_track_genres(user): diff --git a/api/views.py b/api/views.py index a3dfc27..92539d4 100644 --- a/api/views.py +++ b/api/views.py @@ -12,13 +12,100 @@ from datetime import datetime from django.http import JsonResponse from django.db.models import Count, Q -from .utils import parse_library, get_artists_in_genre, update_track_genres +from .utils import get_artists_in_genre, update_track_genres from .models import User, Track, AudioFeatures, Artist # }}} imports # TRACKS_TO_QUERY = 200 +# parse_library {{{ # + +def parse_library(headers, tracks, user): + """Scans user's library for certain number of tracks and store the information in a database + + :headers: For API call. + :tracks: Number of tracks to get from user's library. + :user: a User object representing the user whose library we are parsing + + :returns: None + + """ + # TODO: implement importing entire library with 0 as tracks param + # keeps track of point to get songs from + offset = 0 + payload = {'limit': str(USER_TRACKS_LIMIT)} + artist_genre_queue = [] + features_queue = [] + + # iterate until hit requested num of tracks + for i in range(0, tracks, USER_TRACKS_LIMIT): + payload['offset'] = str(offset) + saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', + headers=headers, + params=payload).json() + + for track_dict in saved_tracks_response['items']: + # add artists {{{ # + + # update artist info before track so that Track object can reference + # Artist object + track_artists = [] + for artist_dict in track_dict['track']['artists']: + artist_obj, artist_created = Artist.objects.get_or_create( + artist_id=artist_dict['id'], + name=artist_dict['name'],) + # only add/tally up artist genres if new + if artist_created: + artist_genre_queue.append(artist_obj) + if len(artist_genre_queue) == ARTIST_LIMIT: + add_artist_genres(headers, artist_genre_queue) + artist_genre_queue = [] + track_artists.append(artist_obj) + + # }}} add artists # + + # TODO: fix this, don't need any more + top_genre = "" + track_obj, track_created = save_track_obj(track_dict['track'], + track_artists, top_genre, user) + + # add audio features {{{ # + + # if a new track is not created, the associated audio feature does + # not need to be created again + if track_created: + features_queue.append(track_obj) + if len(features_queue) == FEATURES_LIMIT: + get_audio_features(headers, features_queue) + features_queue = [] + + # }}} add audio features # + + # temporary console logging + print("#{}-{}: {} - {}".format(offset + 1, + offset + USER_TRACKS_LIMIT, + track_obj.artists.first(), + track_obj.name)) + + # calculates num_songs with offset + songs retrieved + offset += USER_TRACKS_LIMIT + + # clean-up {{{ # + + # update remaining artists without genres and songs without features if + # there are any + if len(artist_genre_queue) > 0: + add_artist_genres(headers, artist_genre_queue) + if len(features_queue) > 0: + get_audio_features(headers, features_queue) + + # }}} clean-up # + + update_track_genres(user) + +# }}} parse_library # + # get_artist_data {{{ # diff --git a/graphs/urls.py b/graphs/urls.py index 9bfcf99..0453083 100644 --- a/graphs/urls.py +++ b/graphs/urls.py @@ -2,6 +2,7 @@ from django.urls import path, include from .views import * +app_name = 'graphs' urlpatterns = [ path('artists/', artist_data, name='display_artist_graph'), diff --git a/login/models.py b/login/models.py new file mode 100644 index 0000000..e3fa787 --- /dev/null +++ b/login/models.py @@ -0,0 +1,22 @@ +from django.db import models + +# id's are 22 in length in examples but set to 30 for buffer +MAX_ID = 30 +# saw tokens being about ~150 chars in length +TOKEN_LENGTH = 200 + +class User(models.Model): + class Meta: + verbose_name = "User" + verbose_name_plural = "Users" + + # the user's Spotify ID + id = models.CharField(primary_key=True, max_length=MAX_ID) + secret = models.CharField(max_length=50, default='') + refresh_token = models.CharField(max_length=TOKEN_LENGTH) + access_token = models.CharField(max_length=TOKEN_LENGTH) + access_obtained_at = models.DateTimeField(auto_now_add=True) + access_expires_in = models.PositiveIntegerField() + + def __str__(self): + return self.user_id diff --git a/login/templates/login/index.html b/login/templates/login/index.html index 9f6e27a..3a41ec5 100644 --- a/login/templates/login/index.html +++ b/login/templates/login/index.html @@ -4,7 +4,7 @@ User Login - +