diff --git a/musicvis/__init__.py b/api/__init__.py similarity index 100% rename from musicvis/__init__.py rename to api/__init__.py diff --git a/spotifyvis/admin.py b/api/admin.py similarity index 100% rename from spotifyvis/admin.py rename to api/admin.py diff --git a/api/apps.py b/api/apps.py new file mode 100644 index 0000000..c147de3 --- /dev/null +++ b/api/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class ApiConfig(AppConfig): + name = 'api' diff --git a/spotifyvis/models.py b/api/models.py similarity index 80% rename from spotifyvis/models.py rename to api/models.py index 7419443..41daa01 100644 --- a/spotifyvis/models.py +++ b/api/models.py @@ -1,4 +1,5 @@ from django.db import models +from login.models import User # id's are 22 in length in examples but set to 30 for buffer MAX_ID = 30 @@ -21,15 +22,13 @@ class Genre(models.Model): # Artist {{{ # - class Artist(models.Model): class Meta: verbose_name = "Artist" verbose_name_plural = "Artists" - artist_id = models.CharField(primary_key=True, max_length=MAX_ID) - # unique since only storing one genre per artist right now - name = models.CharField(unique=True, max_length=50) + id = models.CharField(primary_key=True, max_length=MAX_ID) + name = models.CharField(max_length=50) genres = models.ManyToManyField(Genre, blank=True) def __str__(self): @@ -37,21 +36,6 @@ class Artist(models.Model): # }}} Artist # -# User {{{ # - -class User(models.Model): - class Meta: - verbose_name = "User" - verbose_name_plural = "Users" - - user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID - user_secret = models.CharField(max_length=50, default='') - - def __str__(self): - return self.user_id - -# }}} User # - # Track {{{ # class Track(models.Model): @@ -60,7 +44,7 @@ class Track(models.Model): verbose_name = "Track" verbose_name_plural = "Tracks" - track_id = models.CharField(primary_key=True, max_length=MAX_ID) + id = models.CharField(primary_key=True, max_length=MAX_ID) # artist = models.ForeignKey(Artist, on_delete=models.CASCADE) artists = models.ManyToManyField(Artist, blank=True) year = models.PositiveSmallIntegerField() diff --git a/api/urls.py b/api/urls.py new file mode 100644 index 0000000..60126f6 --- /dev/null +++ b/api/urls.py @@ -0,0 +1,15 @@ +from django.urls import path, include + +from .views import * + +app_name = 'api' +urlpatterns = [ + path('scan/', parse_library, + name='scan'), + path('user_artists/', get_artist_data, + name='get_artist_data'), + path('user_genres/', get_genre_data, + name='get_genre_data'), + path('audio_features//', + get_audio_feature_data, name='get_audio_feature_data'), +] diff --git a/spotifyvis/utils.py b/api/utils.py similarity index 57% rename from spotifyvis/utils.py rename to api/utils.py index 668c274..d9db42e 100644 --- a/spotifyvis/utils.py +++ b/api/utils.py @@ -2,120 +2,36 @@ import requests import math import pprint +import os +import json -from .models import * from django.db.models import Count, Q, F from django.http import JsonResponse from django.core import serializers -import json +from django.utils import timezone +from .models import * +from login.models import User # }}} imports # -USER_TRACKS_LIMIT = 50 -ARTIST_LIMIT = 50 -FEATURES_LIMIT = 100 -# ARTIST_LIMIT = 25 -# FEATURES_LIMIT = 25 - -# parse_library {{{ # - -def parse_library(headers, tracks, user): - """Scans user's library for certain number of tracks and store the information in a database - - :headers: For API call. - :tracks: Number of tracks to get from user's library. - :user: a User object representing the user whose library we are parsing - - :returns: None - - """ - # TODO: implement importing entire library with 0 as tracks param - # keeps track of point to get songs from - offset = 0 - payload = {'limit': str(USER_TRACKS_LIMIT)} - artist_genre_queue = [] - features_queue = [] - - # iterate until hit requested num of tracks - for i in range(0, tracks, USER_TRACKS_LIMIT): - payload['offset'] = str(offset) - saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', - headers=headers, - params=payload).json() - - for track_dict in saved_tracks_response['items']: - # add artists {{{ # - - # update artist info before track so that Track object can reference - # Artist object - track_artists = [] - for artist_dict in track_dict['track']['artists']: - artist_obj, artist_created = Artist.objects.get_or_create( - artist_id=artist_dict['id'], - name=artist_dict['name'],) - # only add/tally up artist genres if new - if artist_created: - artist_genre_queue.append(artist_obj) - if len(artist_genre_queue) == ARTIST_LIMIT: - add_artist_genres(headers, artist_genre_queue) - artist_genre_queue = [] - track_artists.append(artist_obj) - - # }}} add artists # - - # TODO: fix this, don't need any more - top_genre = "" - track_obj, track_created = save_track_obj(track_dict['track'], - track_artists, top_genre, user) - - # add audio features {{{ # - - # if a new track is not created, the associated audio feature does - # not need to be created again - if track_created: - features_queue.append(track_obj) - if len(features_queue) == FEATURES_LIMIT: - get_audio_features(headers, features_queue) - features_queue = [] - - # }}} add audio features # - - # temporary console logging - print("#{}-{}: {} - {}".format(offset + 1, - offset + USER_TRACKS_LIMIT, - track_obj.artists.first(), - track_obj.name)) - - # calculates num_songs with offset + songs retrieved - offset += USER_TRACKS_LIMIT - - # clean-up {{{ # - - # update remaining artists without genres and songs without features if - # there are any - if len(artist_genre_queue) > 0: - add_artist_genres(headers, artist_genre_queue) - if len(features_queue) > 0: - get_audio_features(headers, features_queue) - - # }}} clean-up # - - update_track_genres(user) - -# }}} parse_library # +console_logging = True +# console_logging = False +artists_genre_processed = 0 +features_processed = 0 # update_track_genres {{{ # -def update_track_genres(user): - """Updates user's tracks with the most common genre associated with the +def update_track_genres(user_obj): + """Updates user_obj's tracks with the most common genre associated with the songs' artist(s). - :user: User object who's tracks are being updated. + :user_obj: User object who's tracks are being updated. :returns: None """ - user_tracks = Track.objects.filter(users__exact=user) + tracks_processed = 0 + user_tracks = Track.objects.filter(users__exact=user_obj) for track in user_tracks: # just using this variable to save another call to db track_artists = track.artists.all() @@ -131,41 +47,46 @@ def update_track_genres(user): track.genre = most_common_genre if most_common_genre is not None \ else undefined_genre_obj track.save() - # print(track.name, track.genre) + tracks_processed += 1 + + if console_logging: + print("Added '{}' as genre for song #{} - '{}'".format( + track.genre, + tracks_processed, + track.name, + )) # }}} update_track_genres # # save_track_obj {{{ # -def save_track_obj(track_dict, artists, top_genre, user): +def save_track_obj(track_dict, artists, user_obj): """Make an entry in the database for this track if it doesn't exist already. :track_dict: dictionary from the API call containing track information. :artists: artists of the song, passed in as a list of Artist objects. - :top_genre: top genre associated with this track (see get_top_genre). - :user: User object for which this Track is to be associated with. + :user_obj: User object for which this Track is to be associated with. :returns: (The created/retrieved Track object, created) """ - track_query = Track.objects.filter(track_id__exact=track_dict['id']) + track_query = Track.objects.filter(id__exact=track_dict['id']) if len(track_query) != 0: return track_query[0], False else: new_track = Track.objects.create( - track_id=track_dict['id'], + id=track_dict['id'], year=track_dict['album']['release_date'].split('-')[0], popularity=int(track_dict['popularity']), runtime=int(float(track_dict['duration_ms']) / 1000), name=track_dict['name'], - # genre=top_genre, ) - # have to add artists and user after saving object since track needs to + # have to add artists and user_obj after saving object since track needs to # have ID before filling in m2m field for artist in artists: new_track.artists.add(artist) - new_track.users.add(user) + new_track.users.add(user_obj) new_track.save() return new_track, True @@ -183,13 +104,16 @@ def get_audio_features(headers, track_objs): :returns: None """ - track_ids = str.join(",", [track_obj.track_id for track_obj in track_objs]) + track_ids = str.join(",", [track_obj.id for track_obj in track_objs]) params = {'ids': track_ids} features_response = requests.get("https://api.spotify.com/v1/audio-features", - headers=headers,params=params).json()['audio_features'] + headers=headers, + params={'ids': track_ids} + ).json()['audio_features'] # pprint.pprint(features_response) - useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", ] + useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", + "track_href", "analysis_url", "time_signature", ] for i in range(len(track_objs)): if features_response[i] is not None: # Data that we don't need @@ -200,8 +124,16 @@ def get_audio_features(headers, track_objs): setattr(cur_features_obj, key, val) cur_features_obj.save() + if console_logging: + global features_processed + features_processed += 1 + print("Added features for song #{} - {}".format( + features_processed, track_objs[i].name)) + # }}} get_audio_features # +# process_artist_genre {{{ # + def process_artist_genre(genre_name, artist_obj): """Increase count for correspoding Genre object to genre_name and add that Genre to artist_obj. @@ -219,6 +151,8 @@ def process_artist_genre(genre_name, artist_obj): artist_obj.genres.add(genre_obj) artist_obj.save() +# }}} process_artist_genre # + # add_artist_genres {{{ # def add_artist_genres(headers, artist_objs): @@ -232,11 +166,12 @@ def add_artist_genres(headers, artist_objs): :returns: None """ - artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs]) + artist_ids = str.join(",", [artist_obj.id for artist_obj in artist_objs]) params = {'ids': artist_ids} artists_response = requests.get('https://api.spotify.com/v1/artists/', - headers=headers, params=params).json()['artists'] - # pprint.pprint(artists_response) + headers=headers, + params={'ids': artist_ids}, + ).json()['artists'] for i in range(len(artist_objs)): if len(artists_response[i]['genres']) == 0: process_artist_genre("undefined", artist_objs[i]) @@ -244,6 +179,12 @@ def add_artist_genres(headers, artist_objs): for genre in artists_response[i]['genres']: process_artist_genre(genre, artist_objs[i]) + if console_logging: + global artists_genre_processed + artists_genre_processed += 1 + print("Added genres for artist #{} - {}".format( + artists_genre_processed, artist_objs[i].name)) + # }}} add_artist_genres # # get_artists_in_genre {{{ # @@ -279,3 +220,29 @@ def get_artists_in_genre(user, genre, max_songs): return processed_artist_counts # }}} get_artists_in_genre # + +def get_user_header(user_obj): + """Returns the authorization string needed to make an API call. + + :user_obj: User to return the auth string for. + :returns: the authorization string used for the header in a Spotify API + call. + + """ + seconds_elapsed = (timezone.now() - + user_obj.access_obtained_at).total_seconds() + if seconds_elapsed >= user_obj.access_expires_in: + req_body = { + 'grant_type': 'refresh_token', + 'refresh_token': user_obj.refresh_token, + 'client_id': os.environ['SPOTIFY_CLIENT_ID'], + 'client_secret': os.environ['SPOTIFY_CLIENT_SECRET'] + } + + token_response = requests.post('https://accounts.spotify.com/api/token', + data=req_body).json() + user_obj.access_token = token_response['access_token'] + user_obj.access_expires_in = token_response['expires_in'] + user_obj.save() + + return {'Authorization': "Bearer " + user_obj.access_token} diff --git a/api/views.py b/api/views.py new file mode 100644 index 0000000..5a482d2 --- /dev/null +++ b/api/views.py @@ -0,0 +1,186 @@ +# imports {{{ # + +import math +import random +import requests +import urllib +import secrets +import pprint +import string + +from django.shortcuts import render, redirect +from django.http import JsonResponse +from django.db.models import Count, Q +from .utils import * +from .models import * +from login.models import User +from login.utils import get_user_context + +# }}} imports # + +USER_TRACKS_LIMIT = 50 +ARTIST_LIMIT = 50 +FEATURES_LIMIT = 100 +# ARTIST_LIMIT = 25 +# FEATURES_LIMIT = 25 +TRACKS_TO_QUERY = 100 + +console_logging = True + +# parse_library {{{ # + +def parse_library(request, user_secret): + """Scans user's library for num_tracks and store the information in a + database. + + :user_secret: secret for User object who's library is being scanned. + :returns: None + """ + + offset = 0 + payload = {'limit': str(USER_TRACKS_LIMIT)} + artist_genre_queue = [] + features_queue = [] + user_obj = User.objects.get(secret=user_secret) + user_headers = get_user_header(user_obj) + + # create this obj so loop runs at least once + saved_tracks_response = [0] + # scan until reach num_tracks or no tracks left if scanning entire library + while (TRACKS_TO_QUERY == 0 or offset < TRACKS_TO_QUERY) and len(saved_tracks_response) > 0: + payload['offset'] = str(offset) + saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', + headers=user_headers, + params=payload).json()['items'] + + if console_logging: + tracks_processed = 0 + + for track_dict in saved_tracks_response: + # add artists {{{ # + + # update artist info before track so that Track object can reference + # Artist object + track_artists = [] + for artist_dict in track_dict['track']['artists']: + artist_obj, artist_created = Artist.objects.get_or_create( + id=artist_dict['id'], + name=artist_dict['name'],) + # only add/tally up artist genres if new + if artist_created: + artist_genre_queue.append(artist_obj) + if len(artist_genre_queue) == ARTIST_LIMIT: + add_artist_genres(user_headers, artist_genre_queue) + artist_genre_queue = [] + track_artists.append(artist_obj) + + # }}} add artists # + + track_obj, track_created = save_track_obj(track_dict['track'], + track_artists, user_obj) + + # add audio features {{{ # + + # if a new track is not created, the associated audio feature does + # not need to be created again + if track_created: + features_queue.append(track_obj) + if len(features_queue) == FEATURES_LIMIT: + get_audio_features(user_headers, features_queue) + features_queue = [] + + # }}} add audio features # + + if console_logging: + tracks_processed += 1 + print("Added track #{}: {} - {}".format( + offset + tracks_processed, + track_obj.artists.first(), + track_obj.name, + )) + + # calculates num_songs with offset + songs retrieved + offset += USER_TRACKS_LIMIT + + # clean-up {{{ # + + # update remaining artists without genres and songs without features if + # there are any + if len(artist_genre_queue) > 0: + add_artist_genres(user_headers, artist_genre_queue) + if len(features_queue) > 0: + get_audio_features(user_headers, features_queue) + + # }}} clean-up # + + update_track_genres(user_obj) + + return render(request, 'graphs/logged_in.html', get_user_context(user_obj)) + +# }}} parse_library # + +# get_artist_data {{{ # + +def get_artist_data(request, user_secret): + """Returns artist data as a JSON serialized list of dictionaries + The (key, value) pairs are (artist name, song count for said artist) + + :param request: the HTTP request + :param user_secret: the user secret used for identification + :return: a JsonResponse + """ + user = User.objects.get(secret=user_secret) + artist_counts = Artist.objects.annotate(num_songs=Count('track', + filter=Q(track__users=user))) + processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} + for artist in artist_counts] + pprint.pprint(processed_artist_counts) + return JsonResponse(data=processed_artist_counts, safe=False) + +# }}} get_artist_data # + +# get_audio_feature_data {{{ # + +def get_audio_feature_data(request, audio_feature, user_secret): + """Returns all data points for a given audio feature + + Args: + request: the HTTP request + audio_feature: The audio feature to be queried + user_secret: client secret, used to identify the user + """ + user = User.objects.get(secret=user_secret) + user_tracks = Track.objects.filter(users=user) + response_payload = { + 'data_points': [], + } + for track in user_tracks: + try: + audio_feature_obj = AudioFeatures.objects.get(track=track) + response_payload['data_points'].append(getattr(audio_feature_obj, audio_feature)) + except AudioFeatures.DoesNotExist: + continue + return JsonResponse(response_payload) + +# }}} get_audio_feature_data # + +# get_genre_data {{{ # + +def get_genre_data(request, user_secret): + """Return genre data needed to create the graph user. + TODO + """ + user = User.objects.get(secret=user_secret) + genre_counts = (Track.objects.filter(users__exact=user) + .values('genre') + .order_by('genre') + .annotate(num_songs=Count('genre')) + ) + for genre_dict in genre_counts: + genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre'], + genre_dict['num_songs']) + print("*** Genre Breakdown ***") + pprint.pprint(list(genre_counts)) + return JsonResponse(data=list(genre_counts), safe=False) + +# }}} get_genre_data # diff --git a/spotifyvis/migrations/__init__.py b/graphs/__init__.py similarity index 100% rename from spotifyvis/migrations/__init__.py rename to graphs/__init__.py diff --git a/graphs/apps.py b/graphs/apps.py new file mode 100644 index 0000000..6b4b056 --- /dev/null +++ b/graphs/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class GraphsConfig(AppConfig): + name = 'graphs' diff --git a/spotifyvis/static/spotifyvis/scripts/artist_graph.js b/graphs/static/graphs/scripts/artist_graph.js similarity index 100% rename from spotifyvis/static/spotifyvis/scripts/artist_graph.js rename to graphs/static/graphs/scripts/artist_graph.js diff --git a/spotifyvis/static/spotifyvis/scripts/audio_feat_graph.js b/graphs/static/graphs/scripts/audio_feat_graph.js similarity index 100% rename from spotifyvis/static/spotifyvis/scripts/audio_feat_graph.js rename to graphs/static/graphs/scripts/audio_feat_graph.js diff --git a/spotifyvis/static/spotifyvis/scripts/genre_graph.js b/graphs/static/graphs/scripts/genre_graph.js similarity index 100% rename from spotifyvis/static/spotifyvis/scripts/genre_graph.js rename to graphs/static/graphs/scripts/genre_graph.js diff --git a/spotifyvis/templates/spotifyvis/artist_graph.html b/graphs/templates/graphs/artist_graph.html similarity index 66% rename from spotifyvis/templates/spotifyvis/artist_graph.html rename to graphs/templates/graphs/artist_graph.html index 0e5d49a..78a682c 100644 --- a/spotifyvis/templates/spotifyvis/artist_graph.html +++ b/graphs/templates/graphs/artist_graph.html @@ -6,11 +6,10 @@ Artist Graphs -

Logged in as {{ user_id }}

- + - \ No newline at end of file + diff --git a/spotifyvis/templates/spotifyvis/audio_features.html b/graphs/templates/graphs/features_graphs.html similarity index 93% rename from spotifyvis/templates/spotifyvis/audio_features.html rename to graphs/templates/graphs/features_graphs.html index f4eaafb..ea01793 100644 --- a/spotifyvis/templates/spotifyvis/audio_features.html +++ b/graphs/templates/graphs/features_graphs.html @@ -20,9 +20,8 @@ -

Logged in as {{ user_id }}

- + {% load static %} - + diff --git a/spotifyvis/templates/spotifyvis/logged_in.html b/graphs/templates/graphs/logged_in.html similarity index 57% rename from spotifyvis/templates/spotifyvis/logged_in.html rename to graphs/templates/graphs/logged_in.html index 1df0826..d553f92 100644 --- a/spotifyvis/templates/spotifyvis/logged_in.html +++ b/graphs/templates/graphs/logged_in.html @@ -5,15 +5,15 @@ Logged In - +

{{ user_id }}'s Graphs

- Audio Features - Genres - + Artists diff --git a/graphs/urls.py b/graphs/urls.py new file mode 100644 index 0000000..e1bf0b2 --- /dev/null +++ b/graphs/urls.py @@ -0,0 +1,13 @@ +from django.urls import path, include + +from .views import * + +app_name = 'graphs' +urlpatterns = [ + path('artists/', display_artist_graph, + name='display_artist_graph'), + path('genre/', display_genre_graph, + name='display_genre_graph'), + path('audio_features/', display_features_graphs, + name='display_audio_features'), +] diff --git a/graphs/utils.py b/graphs/utils.py new file mode 100644 index 0000000..d2cf67c --- /dev/null +++ b/graphs/utils.py @@ -0,0 +1,8 @@ +def get_secret_context(user_secret): + """Return user_secret in context for graph pages. + + :user_secret: User secret to put in context. + :returns: context with user secret. + + """ + return { 'user_secret': user_secret, } diff --git a/graphs/views.py b/graphs/views.py new file mode 100644 index 0000000..25c8177 --- /dev/null +++ b/graphs/views.py @@ -0,0 +1,42 @@ +# imports {{{ # + +import math +import random +import requests +import os +import urllib +import secrets +import pprint +import string +from datetime import datetime + +from django.shortcuts import render, redirect +from .utils import * + +# }}} imports # + +def display_artist_graph(request, user_secret): + """Renders the artist data graph display page + + :param request: the HTTP request + :param user_secret: the user secret used for identification + :return: render the artist data graph display page + """ + return render(request, "graphs/artist_graph.html", + get_secret_context(user_secret)) + + +def display_genre_graph(request, user_secret): + return render(request, "graphs/genre_graph.html", + get_secret_context(user_secret)) + + +def display_features_graphs(request, user_secret): + """Renders the audio features page + + :param request: the HTTP request + :param user_secret: user secret used for identification + :return: renders the audio features page + """ + return render(request, "graphs/features_graphs.html", + get_secret_context(user_secret)) diff --git a/login/__init__.py b/login/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/login/apps.py b/login/apps.py new file mode 100644 index 0000000..645de5f --- /dev/null +++ b/login/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class LoginConfig(AppConfig): + name = 'login' diff --git a/login/models.py b/login/models.py new file mode 100644 index 0000000..aa04baf --- /dev/null +++ b/login/models.py @@ -0,0 +1,22 @@ +from django.db import models + +# id's are 22 in length in examples but set to 30 for buffer +MAX_ID = 30 +# saw tokens being about ~150 chars in length +TOKEN_LENGTH = 200 + +class User(models.Model): + class Meta: + verbose_name = "User" + verbose_name_plural = "Users" + + # the user's Spotify ID + id = models.CharField(primary_key=True, max_length=MAX_ID) + secret = models.CharField(max_length=50, default='') + refresh_token = models.CharField(max_length=TOKEN_LENGTH) + access_token = models.CharField(max_length=TOKEN_LENGTH) + access_obtained_at = models.DateTimeField(auto_now=True) + access_expires_in = models.PositiveIntegerField() + + def __str__(self): + return self.id diff --git a/spotifyvis/templates/spotifyvis/index.html b/login/templates/login/index.html similarity index 63% rename from spotifyvis/templates/spotifyvis/index.html rename to login/templates/login/index.html index 9f6e27a..3a41ec5 100644 --- a/spotifyvis/templates/spotifyvis/index.html +++ b/login/templates/login/index.html @@ -4,7 +4,7 @@ User Login - +