From 2644a3aeb211b925bbb63fc5e70271fc512976bc Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Sun, 4 Nov 2018 18:38:25 -0500 Subject: [PATCH 01/20] Parse listening history and add Track objects --- api/models.py | 2 +- api/urls.py | 6 ++- api/utils.py | 43 +++++++++++---- api/views.py | 68 ++++++++++++++++++++++++ graphs/templates/graphs/genre_graph.html | 21 +++++++- login/templates/login/scan.html | 5 +- login/views.py | 3 +- 7 files changed, 131 insertions(+), 17 deletions(-) diff --git a/api/models.py b/api/models.py index 41daa01..4573f94 100644 --- a/api/models.py +++ b/api/models.py @@ -47,7 +47,7 @@ class Track(models.Model): id = models.CharField(primary_key=True, max_length=MAX_ID) # artist = models.ForeignKey(Artist, on_delete=models.CASCADE) artists = models.ManyToManyField(Artist, blank=True) - year = models.PositiveSmallIntegerField() + year = models.PositiveSmallIntegerField(null=True) popularity = models.PositiveSmallIntegerField() runtime = models.PositiveSmallIntegerField() name = models.CharField(max_length=200) diff --git a/api/urls.py b/api/urls.py index 60126f6..2d9d78b 100644 --- a/api/urls.py +++ b/api/urls.py @@ -4,8 +4,10 @@ from .views import * app_name = 'api' urlpatterns = [ - path('scan/', parse_library, - name='scan'), + path('scan/library/', parse_library, + name='scan_library'), + path('scan/history/', parse_history, + name='scan_history'), path('user_artists/', get_artist_data, name='get_artist_data'), path('user_genres/', get_genre_data, diff --git a/api/utils.py b/api/utils.py index d9db42e..c6a384c 100644 --- a/api/utils.py +++ b/api/utils.py @@ -14,8 +14,8 @@ from login.models import User # }}} imports # -console_logging = True -# console_logging = False +# console_logging = True +console_logging = False artists_genre_processed = 0 features_processed = 0 @@ -74,19 +74,32 @@ def save_track_obj(track_dict, artists, user_obj): if len(track_query) != 0: return track_query[0], False else: - new_track = Track.objects.create( - id=track_dict['id'], - year=track_dict['album']['release_date'].split('-')[0], - popularity=int(track_dict['popularity']), - runtime=int(float(track_dict['duration_ms']) / 1000), - name=track_dict['name'], - ) + # check if track is simple or full, simple Track object won't have year + # if 'album' in track_dict: + try: + new_track = Track.objects.create( + id=track_dict['id'], + year=track_dict['album']['release_date'].split('-')[0], + popularity=int(track_dict['popularity']), + runtime=int(float(track_dict['duration_ms']) / 1000), + name=track_dict['name'], + ) + # else: + except KeyError: + new_track = Track.objects.create( + id=track_dict['id'], + popularity=int(track_dict['popularity']), + runtime=int(float(track_dict['duration_ms']) / 1000), + name=track_dict['name'], + ) # have to add artists and user_obj after saving object since track needs to # have ID before filling in m2m field for artist in artists: new_track.artists.add(artist) - new_track.users.add(user_obj) + # print(new_track.name, artist.name) + if user_obj != None: + new_track.users.add(user_obj) new_track.save() return new_track, True @@ -178,6 +191,7 @@ def add_artist_genres(headers, artist_objs): else: for genre in artists_response[i]['genres']: process_artist_genre(genre, artist_objs[i]) + # print(artist_objs[i].name, genre) if console_logging: global artists_genre_processed @@ -221,6 +235,15 @@ def get_artists_in_genre(user, genre, max_songs): # }}} get_artists_in_genre # +def create_artist_for_track(artist_dict): + """TODO: Docstring for create_artist_for_track. + + :artist_dict: TODO + :returns: None + + """ + pass + def get_user_header(user_obj): """Returns the authorization string needed to make an API call. diff --git a/api/views.py b/api/views.py index 5a482d2..bdcc248 100644 --- a/api/views.py +++ b/api/views.py @@ -19,13 +19,16 @@ from login.utils import get_user_context # }}} imports # USER_TRACKS_LIMIT = 50 +HISTORY_LIMIT = 50 ARTIST_LIMIT = 50 FEATURES_LIMIT = 100 # ARTIST_LIMIT = 25 # FEATURES_LIMIT = 25 TRACKS_TO_QUERY = 100 +HISTORY_ENDPOINT = 'https://api.spotify.com/v1/me/player/recently-played' console_logging = True +# console_logging = False # parse_library {{{ # @@ -119,6 +122,71 @@ def parse_library(request, user_secret): # }}} parse_library # +# parse_history {{{ # + +def parse_history(request, user_secret): + """Scans user's listening history and stores the information in a + database. + + :user_secret: secret for User object who's library is being scanned. + :returns: None + """ + + payload = {'limit': str(USER_TRACKS_LIMIT)} + artist_genre_queue = [] + user_obj = User.objects.get(secret=user_secret) + user_headers = get_user_header(user_obj) + + history_response = requests.get(HISTORY_ENDPOINT, + headers=user_headers, + params=payload).json()['items'] + + if console_logging: + tracks_processed = 0 + + for track_dict in history_response: + # add artists {{{ # + + # update artist info before track so that Track object can reference + # Artist object + track_artists = [] + for artist_dict in track_dict['track']['artists']: + artist_obj, artist_created = Artist.objects.get_or_create( + id=artist_dict['id'], + name=artist_dict['name'],) + # only add/tally up artist genres if new + if artist_created: + artist_genre_queue.append(artist_obj) + if len(artist_genre_queue) == ARTIST_LIMIT: + add_artist_genres(user_headers, artist_genre_queue) + artist_genre_queue = [] + track_artists.append(artist_obj) + + # }}} add artists # + + # don't associate history track with User, not necessarily in their + # library + track_obj, track_created = save_track_obj(track_dict['track'], + track_artists, None) + + if console_logging: + tracks_processed += 1 + print("Added track #{}: {} - {}".format( + tracks_processed, + track_obj.artists.first(), + track_obj.name, + )) + + if len(artist_genre_queue) > 0: + add_artist_genres(user_headers, artist_genre_queue) + + # TODO: update track genres from History relation + # update_track_genres(user_obj) + + return render(request, 'graphs/logged_in.html', get_user_context(user_obj)) + +# }}} get_history # + # get_artist_data {{{ # def get_artist_data(request, user_secret): diff --git a/graphs/templates/graphs/genre_graph.html b/graphs/templates/graphs/genre_graph.html index bf7324e..4eb5940 100644 --- a/graphs/templates/graphs/genre_graph.html +++ b/graphs/templates/graphs/genre_graph.html @@ -21,10 +21,18 @@ + {% load static %} - + + + + diff --git a/login/templates/login/scan.html b/login/templates/login/scan.html index 183742c..e0c6dce 100644 --- a/login/templates/login/scan.html +++ b/login/templates/login/scan.html @@ -18,8 +18,11 @@

You are using an outdated browser. Please upgrade your browser to improve your experience.

Logged in as {{ user_id }}

- + Scan Library + + Scan History + diff --git a/login/views.py b/login/views.py index 9406d19..1e0bcbe 100644 --- a/login/views.py +++ b/login/views.py @@ -19,6 +19,7 @@ from .utils import * TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' TRACKS_TO_QUERY = 200 +AUTH_SCOPE = ['user-library-read', 'user-read-recently-played',] # generate_random_string {{{ # @@ -62,7 +63,7 @@ def spotify_login(request): 'response_type': 'code', 'redirect_uri': 'http://localhost:8000/login/callback', 'state': state_str, - 'scope': 'user-library-read', + 'scope': " ".join(AUTH_SCOPE), 'show_dialog': False } From d06e5912cca0c99970552c595c800baa32c647ff Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Sun, 4 Nov 2018 20:43:55 -0500 Subject: [PATCH 02/20] Create History relation and add entries Logs User, time and Track. --- api/models.py | 20 ++++++++++++++++++++ api/views.py | 12 +++++++++--- requirements.txt | 1 + 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/api/models.py b/api/models.py index 4573f94..a822029 100644 --- a/api/models.py +++ b/api/models.py @@ -86,3 +86,23 @@ class AudioFeatures(models.Model): return super(AudioFeatures, self).__str__() # }}} AudioFeatures # + +# History {{{ # + +class History(models.Model): + + class Meta: + verbose_name = "History" + verbose_name_plural = "History" + unique_together = (("user", "time"),) + + history_id = models.AutoField(primary_key=True) + user = models.ForeignKey(User, on_delete=models.CASCADE) + time = models.DateTimeField() + track = models.ForeignKey(Track, on_delete=models.CASCADE) + + def __str__(self): + return (self.user, self.time, self.track) + +# }}} # + diff --git a/api/views.py b/api/views.py index bdcc248..eb32029 100644 --- a/api/views.py +++ b/api/views.py @@ -15,6 +15,7 @@ from .utils import * from .models import * from login.models import User from login.utils import get_user_context +from dateutil.parser import parse # }}} imports # @@ -168,13 +169,18 @@ def parse_history(request, user_secret): # library track_obj, track_created = save_track_obj(track_dict['track'], track_artists, None) + history_obj, history_created = History.objects.get_or_create( + user=user_obj, + time=parse(track_dict['played_at']), + track=track_obj,) if console_logging: tracks_processed += 1 - print("Added track #{}: {} - {}".format( + print("Added track #{} for user {}: {} - {}".format( tracks_processed, - track_obj.artists.first(), - track_obj.name, + history_obj.user, + history_obj.time, + history_obj.track, )) if len(artist_genre_queue) > 0: diff --git a/requirements.txt b/requirements.txt index 59c9dc6..04ea69f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ requests==2.18.4 six==1.11.0 urllib3==1.22 wrapt==1.10.11 +python-dateutil==2.7.5 From a399960a49136f48d82d01e846199d297545144d Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Sun, 4 Nov 2018 21:18:50 -0500 Subject: [PATCH 03/20] Only get history after latest stored one --- api/models.py | 4 ++-- api/views.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/models.py b/api/models.py index a822029..0b9b337 100644 --- a/api/models.py +++ b/api/models.py @@ -94,11 +94,11 @@ class History(models.Model): class Meta: verbose_name = "History" verbose_name_plural = "History" - unique_together = (("user", "time"),) + unique_together = (("user", "timestamp"),) history_id = models.AutoField(primary_key=True) user = models.ForeignKey(User, on_delete=models.CASCADE) - time = models.DateTimeField() + timestamp = models.DateTimeField() track = models.ForeignKey(Track, on_delete=models.CASCADE) def __str__(self): diff --git a/api/views.py b/api/views.py index eb32029..21aa2bd 100644 --- a/api/views.py +++ b/api/views.py @@ -10,7 +10,7 @@ import string from django.shortcuts import render, redirect from django.http import JsonResponse -from django.db.models import Count, Q +from django.db.models import Count, Q, Max from .utils import * from .models import * from login.models import User @@ -133,11 +133,11 @@ def parse_history(request, user_secret): :returns: None """ - payload = {'limit': str(USER_TRACKS_LIMIT)} - artist_genre_queue = [] user_obj = User.objects.get(secret=user_secret) + last_time_played = History.objects.filter(user=user_obj).aggregate(Max('timestamp'))['timestamp__max'] + payload = {'limit': str(USER_TRACKS_LIMIT), 'after': last_time_played.isoformat()} + artist_genre_queue = [] user_headers = get_user_header(user_obj) - history_response = requests.get(HISTORY_ENDPOINT, headers=user_headers, params=payload).json()['items'] @@ -171,7 +171,7 @@ def parse_history(request, user_secret): track_artists, None) history_obj, history_created = History.objects.get_or_create( user=user_obj, - time=parse(track_dict['played_at']), + timestamp=parse(track_dict['played_at']), track=track_obj,) if console_logging: @@ -179,7 +179,7 @@ def parse_history(request, user_secret): print("Added track #{} for user {}: {} - {}".format( tracks_processed, history_obj.user, - history_obj.time, + history_obj.timestamp, history_obj.track, )) From b4ffddb24d6afb2308f80c141f1bfcd04fd405a4 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Tue, 6 Nov 2018 12:19:28 -0500 Subject: [PATCH 04/20] Setup new page for user history --- api/views.py | 8 ++------ graphs/templates/graphs/logged_in.html | 3 +++ graphs/templates/graphs/user_history.html | 13 +++++++++++++ graphs/urls.py | 2 ++ graphs/utils.py | 12 ++++++++++++ graphs/views.py | 11 +++++++++++ 6 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 graphs/templates/graphs/user_history.html diff --git a/api/views.py b/api/views.py index 21aa2bd..94d1c13 100644 --- a/api/views.py +++ b/api/views.py @@ -176,12 +176,8 @@ def parse_history(request, user_secret): if console_logging: tracks_processed += 1 - print("Added track #{} for user {}: {} - {}".format( - tracks_processed, - history_obj.user, - history_obj.timestamp, - history_obj.track, - )) + print("Added history track #{}: {}".format( + tracks_processed, history_obj,)) if len(artist_genre_queue) > 0: add_artist_genres(user_headers, artist_genre_queue) diff --git a/graphs/templates/graphs/logged_in.html b/graphs/templates/graphs/logged_in.html index d553f92..a804be1 100644 --- a/graphs/templates/graphs/logged_in.html +++ b/graphs/templates/graphs/logged_in.html @@ -16,5 +16,8 @@ Artists + + History + diff --git a/graphs/templates/graphs/user_history.html b/graphs/templates/graphs/user_history.html new file mode 100644 index 0000000..f07e954 --- /dev/null +++ b/graphs/templates/graphs/user_history.html @@ -0,0 +1,13 @@ + +{% load static %} + + + + User History + + + + +

Your Listening History

+ + diff --git a/graphs/urls.py b/graphs/urls.py index e1bf0b2..3d9c0c8 100644 --- a/graphs/urls.py +++ b/graphs/urls.py @@ -10,4 +10,6 @@ urlpatterns = [ name='display_genre_graph'), path('audio_features/', display_features_graphs, name='display_audio_features'), + path('history/', display_history_table, + name='display_history_table'), ] diff --git a/graphs/utils.py b/graphs/utils.py index d2cf67c..e866585 100644 --- a/graphs/utils.py +++ b/graphs/utils.py @@ -6,3 +6,15 @@ def get_secret_context(user_secret): """ return { 'user_secret': user_secret, } + + +def get_user_history(user_secret): + """Return all stored history for corresponding User to user_secret. + + :user_secret: User secret to get history for. + :returns: list of lists of song history plus information. + + """ + # TODO: pass back user id as well? + pass + diff --git a/graphs/views.py b/graphs/views.py index 25c8177..f4d6b44 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -40,3 +40,14 @@ def display_features_graphs(request, user_secret): """ return render(request, "graphs/features_graphs.html", get_secret_context(user_secret)) + +def display_history_table(request, user_secret): + """Renders the user history page + + :param request: the HTTP request + :param user_secret: user secret used for identification + :return: renders the user history page + """ + return render(request, "graphs/user_history.html", + get_secret_context(user_secret)) + From 29129779928aee62ae0979b1e279e6928b19d596 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Tue, 6 Nov 2018 15:23:43 -0500 Subject: [PATCH 05/20] Setup django_tables2 for user history table --- api/models.py | 2 +- api/views.py | 4 ++++ graphs/templates/graphs/user_history.html | 4 +++- graphs/utils.py | 28 +++++++++++++++++++++-- graphs/views.py | 9 +++++--- requirements.txt | 3 ++- spotifyvis/settings.py | 1 + static/css/dark_bg.css | 13 ++++++++--- 8 files changed, 53 insertions(+), 11 deletions(-) diff --git a/api/models.py b/api/models.py index 0b9b337..08fdcd8 100644 --- a/api/models.py +++ b/api/models.py @@ -102,7 +102,7 @@ class History(models.Model): track = models.ForeignKey(Track, on_delete=models.CASCADE) def __str__(self): - return (self.user, self.time, self.track) + return " - ".join((str(self.user), str(self.timestamp), str(self.track))) # }}} # diff --git a/api/views.py b/api/views.py index 94d1c13..3e16ab7 100644 --- a/api/views.py +++ b/api/views.py @@ -19,6 +19,8 @@ from dateutil.parser import parse # }}} imports # +# constants {{{ # + USER_TRACKS_LIMIT = 50 HISTORY_LIMIT = 50 ARTIST_LIMIT = 50 @@ -31,6 +33,8 @@ HISTORY_ENDPOINT = 'https://api.spotify.com/v1/me/player/recently-played' console_logging = True # console_logging = False +# }}} constants # + # parse_library {{{ # def parse_library(request, user_secret): diff --git a/graphs/templates/graphs/user_history.html b/graphs/templates/graphs/user_history.html index f07e954..6086518 100644 --- a/graphs/templates/graphs/user_history.html +++ b/graphs/templates/graphs/user_history.html @@ -1,5 +1,6 @@ {% load static %} +{% load render_table from django_tables2 %} @@ -8,6 +9,7 @@ -

Your Listening History

+

{{ user_id }}'s Listening History

+ {% render_table user_history_table %} diff --git a/graphs/utils.py b/graphs/utils.py index e866585..98c90fd 100644 --- a/graphs/utils.py +++ b/graphs/utils.py @@ -1,3 +1,14 @@ +import django_tables2 as tables + +from pprint import pprint +from login.models import User +from api.models import History + +class HistoryTable(tables.Table): + class Meta: + model = History + template_name = 'django_tables2/bootstrap.html' + def get_secret_context(user_secret): """Return user_secret in context for graph pages. @@ -15,6 +26,19 @@ def get_user_history(user_secret): :returns: list of lists of song history plus information. """ - # TODO: pass back user id as well? - pass + user_id = get_user_id_from_secret(user_secret) + history_fields = [field.name for field in History._meta.get_fields()] + user_history = History.objects.filter(user__exact=user_id).order_by('-timestamp') + user_history_table = HistoryTable(user_history) + return { 'user_id': user_id, + 'history_fields': history_fields, + 'user_history_table': user_history_table, } +def get_user_id_from_secret(user_secret): + """Retrieve corresponding user_id for user_secret. + + :user_secret: + :returns: user_id + + """ + return User.objects.get(secret=user_secret).id diff --git a/graphs/views.py b/graphs/views.py index f4d6b44..0415363 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -6,12 +6,13 @@ import requests import os import urllib import secrets -import pprint +from pprint import pprint import string from datetime import datetime from django.shortcuts import render, redirect from .utils import * +from django_tables2 import RequestConfig # }}} imports # @@ -48,6 +49,8 @@ def display_history_table(request, user_secret): :param user_secret: user secret used for identification :return: renders the user history page """ - return render(request, "graphs/user_history.html", - get_secret_context(user_secret)) + context = get_secret_context(user_secret) + context.update(get_user_history(user_secret)) + RequestConfig(request).configure(context['user_history_table']) + return render(request, "graphs/user_history.html", context) diff --git a/requirements.txt b/requirements.txt index 04ea69f..6bde00c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,15 +3,16 @@ certifi==2018.4.16 chardet==3.0.4 Django==2.0.5 djangorestframework==3.8.2 +django-tables2==2.0.2 idna==2.6 isort==4.3.4 lazy-object-proxy==1.3.1 mccabe==0.6.1 psycopg2-binary==2.7.4 pylint==1.8.4 +python-dateutil==2.7.5 pytz==2018.4 requests==2.18.4 six==1.11.0 urllib3==1.22 wrapt==1.10.11 -python-dateutil==2.7.5 diff --git a/spotifyvis/settings.py b/spotifyvis/settings.py index ffbffaa..d91d2ea 100644 --- a/spotifyvis/settings.py +++ b/spotifyvis/settings.py @@ -40,6 +40,7 @@ INSTALLED_APPS = [ 'login.apps.LoginConfig', 'api.apps.ApiConfig', 'graphs.apps.GraphsConfig', + 'django_tables2', ] MIDDLEWARE = [ diff --git a/static/css/dark_bg.css b/static/css/dark_bg.css index a472959..52af531 100644 --- a/static/css/dark_bg.css +++ b/static/css/dark_bg.css @@ -1,8 +1,15 @@ body { -background-color: #1e1e1e; + /* dark grey */ + background-color: #1e1e1e; } -h1,p { -color: grey; +h1 { + /* light grey */ + color: #e5e5e5; +} + +p,td { + /* light-dark grey */ + color: #b2b2b2; } From 4b19c932b05cca18c5a699a36da80c53feab2a0d Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Tue, 6 Nov 2018 16:14:29 -0500 Subject: [PATCH 06/20] Split Track column into name and artists --- api/models.py | 10 ++++++++-- graphs/utils.py | 14 +++++++++++++- static/css/dark_bg.css | 4 ++-- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/api/models.py b/api/models.py index 08fdcd8..f660b3b 100644 --- a/api/models.py +++ b/api/models.py @@ -45,7 +45,6 @@ class Track(models.Model): verbose_name_plural = "Tracks" id = models.CharField(primary_key=True, max_length=MAX_ID) - # artist = models.ForeignKey(Artist, on_delete=models.CASCADE) artists = models.ManyToManyField(Artist, blank=True) year = models.PositiveSmallIntegerField(null=True) popularity = models.PositiveSmallIntegerField() @@ -96,7 +95,7 @@ class History(models.Model): verbose_name_plural = "History" unique_together = (("user", "timestamp"),) - history_id = models.AutoField(primary_key=True) + id = models.AutoField(primary_key=True) user = models.ForeignKey(User, on_delete=models.CASCADE) timestamp = models.DateTimeField() track = models.ForeignKey(Track, on_delete=models.CASCADE) @@ -104,5 +103,12 @@ class History(models.Model): def __str__(self): return " - ".join((str(self.user), str(self.timestamp), str(self.track))) + def get_track_name(self): + return self.track.name + + def get_artists(self): + artist_names = [artist.name for artist in self.track.artists.all()] + return ', '.join(artist_names) + # }}} # diff --git a/graphs/utils.py b/graphs/utils.py index 98c90fd..bb8cb4c 100644 --- a/graphs/utils.py +++ b/graphs/utils.py @@ -2,13 +2,23 @@ import django_tables2 as tables from pprint import pprint from login.models import User -from api.models import History +from api.models import History, Track class HistoryTable(tables.Table): class Meta: model = History template_name = 'django_tables2/bootstrap.html' + track_name = tables.Column(accessor='get_track_name', orderable=False) + artists = tables.Column(accessor='get_artists', orderable=False) + + # def render_track_name(self, record): + # return record.track.name + # return record.user + + # def render_user(self, value): + # return '' + def get_secret_context(user_secret): """Return user_secret in context for graph pages. @@ -28,7 +38,9 @@ def get_user_history(user_secret): """ user_id = get_user_id_from_secret(user_secret) history_fields = [field.name for field in History._meta.get_fields()] + # don't need ordering bc. django-tables2? user_history = History.objects.filter(user__exact=user_id).order_by('-timestamp') + # user_history = History.objects.filter(user__exact=user_id).order_by('-timestamp') user_history_table = HistoryTable(user_history) return { 'user_id': user_id, 'history_fields': history_fields, diff --git a/static/css/dark_bg.css b/static/css/dark_bg.css index 52af531..2154bc6 100644 --- a/static/css/dark_bg.css +++ b/static/css/dark_bg.css @@ -3,9 +3,9 @@ body { background-color: #1e1e1e; } -h1 { +h1,th { /* light grey */ - color: #e5e5e5; + color: #cccccc; } p,td { From b2990b45eea046235911fcfd4b713b5bd80980ef Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Tue, 6 Nov 2018 18:22:27 -0500 Subject: [PATCH 07/20] Filter out unnecessary columns in history table --- api/views.py | 7 +++++-- graphs/utils.py | 36 +----------------------------------- graphs/views.py | 13 ++++++++++--- requirements.txt | 1 + spotifyvis/settings.py | 1 + 5 files changed, 18 insertions(+), 40 deletions(-) diff --git a/api/views.py b/api/views.py index 3e16ab7..741c4c3 100644 --- a/api/views.py +++ b/api/views.py @@ -5,7 +5,6 @@ import random import requests import urllib import secrets -import pprint import string from django.shortcuts import render, redirect @@ -16,6 +15,7 @@ from .models import * from login.models import User from login.utils import get_user_context from dateutil.parser import parse +from pprint import pprint # }}} imports # @@ -138,13 +138,16 @@ def parse_history(request, user_secret): """ user_obj = User.objects.get(secret=user_secret) + payload = {'limit': str(USER_TRACKS_LIMIT)} last_time_played = History.objects.filter(user=user_obj).aggregate(Max('timestamp'))['timestamp__max'] - payload = {'limit': str(USER_TRACKS_LIMIT), 'after': last_time_played.isoformat()} + if last_time_played is not None: + payload['after'] = last_time_played.isoformat() artist_genre_queue = [] user_headers = get_user_header(user_obj) history_response = requests.get(HISTORY_ENDPOINT, headers=user_headers, params=payload).json()['items'] + # pprint(history_response) if console_logging: tracks_processed = 0 diff --git a/graphs/utils.py b/graphs/utils.py index bb8cb4c..62756bb 100644 --- a/graphs/utils.py +++ b/graphs/utils.py @@ -2,7 +2,7 @@ import django_tables2 as tables from pprint import pprint from login.models import User -from api.models import History, Track +from api.models import History class HistoryTable(tables.Table): class Meta: @@ -12,13 +12,6 @@ class HistoryTable(tables.Table): track_name = tables.Column(accessor='get_track_name', orderable=False) artists = tables.Column(accessor='get_artists', orderable=False) - # def render_track_name(self, record): - # return record.track.name - # return record.user - - # def render_user(self, value): - # return '' - def get_secret_context(user_secret): """Return user_secret in context for graph pages. @@ -27,30 +20,3 @@ def get_secret_context(user_secret): """ return { 'user_secret': user_secret, } - - -def get_user_history(user_secret): - """Return all stored history for corresponding User to user_secret. - - :user_secret: User secret to get history for. - :returns: list of lists of song history plus information. - - """ - user_id = get_user_id_from_secret(user_secret) - history_fields = [field.name for field in History._meta.get_fields()] - # don't need ordering bc. django-tables2? - user_history = History.objects.filter(user__exact=user_id).order_by('-timestamp') - # user_history = History.objects.filter(user__exact=user_id).order_by('-timestamp') - user_history_table = HistoryTable(user_history) - return { 'user_id': user_id, - 'history_fields': history_fields, - 'user_history_table': user_history_table, } - -def get_user_id_from_secret(user_secret): - """Retrieve corresponding user_id for user_secret. - - :user_secret: - :returns: user_id - - """ - return User.objects.get(secret=user_secret).id diff --git a/graphs/views.py b/graphs/views.py index 0415363..4bfbcdb 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -13,6 +13,7 @@ from datetime import datetime from django.shortcuts import render, redirect from .utils import * from django_tables2 import RequestConfig +from api.models import History # }}} imports # @@ -49,8 +50,14 @@ def display_history_table(request, user_secret): :param user_secret: user secret used for identification :return: renders the user history page """ - context = get_secret_context(user_secret) - context.update(get_user_history(user_secret)) - RequestConfig(request).configure(context['user_history_table']) + user_id = User.objects.get(secret=user_secret).id + user_history = History.objects.filter(user__exact=user_id).order_by('-timestamp') + history_table = HistoryTable(user_history) + history_table.exclude = ('id', 'user', 'track', ) + RequestConfig(request).configure(history_table) + + context = { 'user_history_table': history_table, + 'user_id': user_id, } + return render(request, "graphs/user_history.html", context) diff --git a/requirements.txt b/requirements.txt index 6bde00c..936771a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ astroid==1.6.3 certifi==2018.4.16 chardet==3.0.4 Django==2.0.5 +django-filter==2.0 djangorestframework==3.8.2 django-tables2==2.0.2 idna==2.6 diff --git a/spotifyvis/settings.py b/spotifyvis/settings.py index d91d2ea..b8db2ef 100644 --- a/spotifyvis/settings.py +++ b/spotifyvis/settings.py @@ -41,6 +41,7 @@ INSTALLED_APPS = [ 'api.apps.ApiConfig', 'graphs.apps.GraphsConfig', 'django_tables2', + 'django_filters', ] MIDDLEWARE = [ From 77141849acd4db015a8b5bb240efaa08578fed69 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Tue, 6 Nov 2018 21:35:14 -0500 Subject: [PATCH 08/20] Move non-request functions in login/views to utils --- graphs/templates/graphs/user_history.html | 1 + graphs/views.py | 8 ++- login/templates/login/index.html | 2 +- login/utils.py | 59 +++++++++++++++++++++++ login/views.py | 57 ++-------------------- 5 files changed, 71 insertions(+), 56 deletions(-) diff --git a/graphs/templates/graphs/user_history.html b/graphs/templates/graphs/user_history.html index 6086518..7d2cd25 100644 --- a/graphs/templates/graphs/user_history.html +++ b/graphs/templates/graphs/user_history.html @@ -10,6 +10,7 @@

{{ user_id }}'s Listening History

+ Export CSV {% render_table user_history_table %} diff --git a/graphs/views.py b/graphs/views.py index 4bfbcdb..6a9811e 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -12,7 +12,7 @@ from datetime import datetime from django.shortcuts import render, redirect from .utils import * -from django_tables2 import RequestConfig +from django_tables2 import RequestConfig, SingleTableView from api.models import History # }}} imports # @@ -61,3 +61,9 @@ def display_history_table(request, user_secret): return render(request, "graphs/user_history.html", context) +class HistoryList(SingleTableView): + """Create table with list of song history.""" + model = History + table_class = HistoryTable + # table_data = + diff --git a/login/templates/login/index.html b/login/templates/login/index.html index 3a41ec5..eb428de 100644 --- a/login/templates/login/index.html +++ b/login/templates/login/index.html @@ -20,7 +20,7 @@

spotify-lib-vis

- Login + Login Admin Graphs
diff --git a/login/utils.py b/login/utils.py index 695bfae..1f2bdef 100644 --- a/login/utils.py +++ b/login/utils.py @@ -1,3 +1,7 @@ +import string +import random +import requests + from .models import User def get_user_context(user_obj): @@ -8,3 +12,58 @@ def get_user_context(user_obj): """ return { 'user_id': user_obj.id, 'user_secret': user_obj.secret, } + + +# generate_random_string {{{ # + +def generate_random_string(length): + """Generates a random string of a certain length + + Args: + length: the desired length of the randomized string + + Returns: + A random string + """ + all_chars = string.ascii_letters + string.digits + rand_str = "".join(random.choice(all_chars) for _ in range(length)) + + return rand_str + +# }}} generate_random_string # + +# create_user {{{ # + + +def create_user(refresh_token, access_token, access_expires_in): + """Create a User object based on information returned from Step 2 (callback + function) of auth flow. + + :refresh_token: Used to renew access tokens. + :access_token: Used in Spotify API calls. + :access_expires_in: How long the access token last in seconds. + + :returns: The newly created User object. + + """ + profile_response = requests.get('https://api.spotify.com/v1/me', + headers={'Authorization': "Bearer " + access_token}).json() + user_id = profile_response['id'] + + try: + user_obj = User.objects.get(id=user_id) + except User.DoesNotExist: + # Python docs recommends 32 bytes of randomness against brute + # force attacks + user_obj = User.objects.create( + id=user_id, + secret=secrets.token_urlsafe(32), + refresh_token=refresh_token, + access_token=access_token, + access_expires_in=access_expires_in, + ) + + return user_obj + +# }}} create_user # + diff --git a/login/views.py b/login/views.py index 1e0bcbe..17db2a5 100644 --- a/login/views.py +++ b/login/views.py @@ -1,13 +1,10 @@ # imports {{{ # import math -import random -import requests import os import urllib import secrets import pprint -import string from datetime import datetime from django.shortcuts import render, redirect @@ -21,28 +18,8 @@ TIME_FORMAT = '%Y-%m-%d-%H-%M-%S' TRACKS_TO_QUERY = 200 AUTH_SCOPE = ['user-library-read', 'user-read-recently-played',] -# generate_random_string {{{ # - - -def generate_random_string(length): - """Generates a random string of a certain length - - Args: - length: the desired length of the randomized string - - Returns: - A random string - """ - all_chars = string.ascii_letters + string.digits - rand_str = "".join(random.choice(all_chars) for _ in range(length)) - - return rand_str - -# }}} generate_random_string # - # index {{{ # -# Create your views here. def index(request): return render(request, 'login/index.html') @@ -73,6 +50,8 @@ def spotify_login(request): # }}} spotify_login # +# callback {{{ # + def callback(request): """ Step 2 in authorization flow: Have your application request refresh and access tokens; Spotify returns access and refresh tokens. @@ -97,38 +76,8 @@ def callback(request): token_response['expires_in']) return render(request, 'login/scan.html', get_user_context(user_obj)) - # return redirect('user/' + user_obj.secret) - -def create_user(refresh_token, access_token, access_expires_in): - """Create a User object based on information returned from Step 2 (callback - function) of auth flow. - - :refresh_token: Used to renew access tokens. - :access_token: Used in Spotify API calls. - :access_expires_in: How long the access token last in seconds. - - :returns: The newly created User object. - - """ - profile_response = requests.get('https://api.spotify.com/v1/me', - headers={'Authorization': "Bearer " + access_token}).json() - user_id = profile_response['id'] - - try: - user_obj = User.objects.get(id=user_id) - except User.DoesNotExist: - # Python docs recommends 32 bytes of randomness against brute - # force attacks - user_obj = User.objects.create( - id=user_id, - secret=secrets.token_urlsafe(32), - refresh_token=refresh_token, - access_token=access_token, - access_expires_in=access_expires_in, - ) - - return user_obj +# }}} callback # # admin_graphs {{{ # From 3d6dff359d75346cd4c6351b6a6890f7c5191908 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Tue, 6 Nov 2018 21:45:52 -0500 Subject: [PATCH 09/20] Store user id/secret in session upon login (#61) History table uses session's user_id instead of secret in URL. --- graphs/templates/graphs/logged_in.html | 2 +- graphs/urls.py | 5 +++-- graphs/views.py | 4 ++-- login/views.py | 7 +++++++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/graphs/templates/graphs/logged_in.html b/graphs/templates/graphs/logged_in.html index a804be1..f34655d 100644 --- a/graphs/templates/graphs/logged_in.html +++ b/graphs/templates/graphs/logged_in.html @@ -16,7 +16,7 @@ Artists - + History diff --git a/graphs/urls.py b/graphs/urls.py index 3d9c0c8..7f985e1 100644 --- a/graphs/urls.py +++ b/graphs/urls.py @@ -10,6 +10,7 @@ urlpatterns = [ name='display_genre_graph'), path('audio_features/', display_features_graphs, name='display_audio_features'), - path('history/', display_history_table, - name='display_history_table'), + # path('history/', display_history_table, + # name='display_history_table'), + path('history/', display_history_table, name='display_history_table'), ] diff --git a/graphs/views.py b/graphs/views.py index 6a9811e..85fa94c 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -43,14 +43,14 @@ def display_features_graphs(request, user_secret): return render(request, "graphs/features_graphs.html", get_secret_context(user_secret)) -def display_history_table(request, user_secret): +def display_history_table(request): """Renders the user history page :param request: the HTTP request :param user_secret: user secret used for identification :return: renders the user history page """ - user_id = User.objects.get(secret=user_secret).id + user_id = request.session['user_id'] user_history = History.objects.filter(user__exact=user_id).order_by('-timestamp') history_table = HistoryTable(user_history) history_table.exclude = ('id', 'user', 'track', ) diff --git a/login/views.py b/login/views.py index 17db2a5..649ced2 100644 --- a/login/views.py +++ b/login/views.py @@ -74,6 +74,9 @@ def callback(request): user_obj = create_user(token_response['refresh_token'], token_response['access_token'], token_response['expires_in']) + + request.session['user_id'] = user_obj.id + request.session['user_secret'] = user_obj.secret return render(request, 'login/scan.html', get_user_context(user_obj)) @@ -86,6 +89,10 @@ def admin_graphs(request): """ user_id = "polarbier" # user_id = "chrisshyi13" + + request.session['user_id'] = user_id + # request.session['user_secret'] = user_obj.secret + request.session['user_secret'] = User.objects.get(id=user_id).secret user_obj = User.objects.get(id=user_id) return render(request, 'graphs/logged_in.html', get_user_context(user_obj)) From fa2c5f700883d37f57d405d59efbfe4b0698d280 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Tue, 6 Nov 2018 22:58:38 -0500 Subject: [PATCH 10/20] Export listening history to csv (#57) Still need to convert timestamps to more CPU-readable format and figure out what to do about reading headers for importing. --- api/models.py | 3 ++ graphs/templates/graphs/user_history.html | 4 +- graphs/urls.py | 3 +- graphs/utils.py | 1 + graphs/views.py | 46 +++++++++++++---------- requirements.txt | 1 + 6 files changed, 36 insertions(+), 22 deletions(-) diff --git a/api/models.py b/api/models.py index f660b3b..be099d0 100644 --- a/api/models.py +++ b/api/models.py @@ -106,6 +106,9 @@ class History(models.Model): def get_track_name(self): return self.track.name + def get_track_id(self): + return self.track.id + def get_artists(self): artist_names = [artist.name for artist in self.track.artists.all()] return ', '.join(artist_names) diff --git a/graphs/templates/graphs/user_history.html b/graphs/templates/graphs/user_history.html index 7d2cd25..8d41a06 100644 --- a/graphs/templates/graphs/user_history.html +++ b/graphs/templates/graphs/user_history.html @@ -1,6 +1,6 @@ {% load static %} -{% load render_table from django_tables2 %} +{% load render_table export_url from django_tables2 %} @@ -10,7 +10,7 @@

{{ user_id }}'s Listening History

- Export CSV + Export CSV {% render_table user_history_table %} diff --git a/graphs/urls.py b/graphs/urls.py index 7f985e1..6ff4da5 100644 --- a/graphs/urls.py +++ b/graphs/urls.py @@ -12,5 +12,6 @@ urlpatterns = [ name='display_audio_features'), # path('history/', display_history_table, # name='display_history_table'), - path('history/', display_history_table, name='display_history_table'), + # path('history/', display_history_table, name='display_history_table'), + path('history/', HistoryList.as_view(), name='display_history_table'), ] diff --git a/graphs/utils.py b/graphs/utils.py index 62756bb..69e262d 100644 --- a/graphs/utils.py +++ b/graphs/utils.py @@ -10,6 +10,7 @@ class HistoryTable(tables.Table): template_name = 'django_tables2/bootstrap.html' track_name = tables.Column(accessor='get_track_name', orderable=False) + track_id = tables.Column(accessor='get_track_id', orderable=False) artists = tables.Column(accessor='get_artists', orderable=False) def get_secret_context(user_secret): diff --git a/graphs/views.py b/graphs/views.py index 85fa94c..37b1c5c 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -13,6 +13,8 @@ from datetime import datetime from django.shortcuts import render, redirect from .utils import * from django_tables2 import RequestConfig, SingleTableView +from django_tables2.export.views import ExportMixin +from django_tables2.export import TableExport from api.models import History # }}} imports # @@ -43,27 +45,33 @@ def display_features_graphs(request, user_secret): return render(request, "graphs/features_graphs.html", get_secret_context(user_secret)) -def display_history_table(request): - """Renders the user history page +class HistoryList(ExportMixin, SingleTableView): + """Create table with list of song history.""" + model = History + table_class = HistoryTable + context_table_name = 'user_history_table' + template_name = 'graphs/user_history.html' - :param request: the HTTP request - :param user_secret: user secret used for identification - :return: renders the user history page - """ - user_id = request.session['user_id'] - user_history = History.objects.filter(user__exact=user_id).order_by('-timestamp') - history_table = HistoryTable(user_history) - history_table.exclude = ('id', 'user', 'track', ) - RequestConfig(request).configure(history_table) + def get_table_kwargs(self): + return { 'exclude': ('id', 'user', 'track', 'track_id', ) } - context = { 'user_history_table': history_table, - 'user_id': user_id, } + def get_table_data(self): + return History.objects.filter(user__exact=self.request.session['user_id']).order_by('-timestamp') - return render(request, "graphs/user_history.html", context) + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['user_id'] = self.request.session['user_id'] + return context -class HistoryList(SingleTableView): - """Create table with list of song history.""" - model = History - table_class = HistoryTable - # table_data = + def get_export_filename(self, export_format): + return "{}.{}".format(self.request.session['user_id'], export_format) + + def create_export(self, export_format): + export_exclude = ('id', 'user', 'track', 'track_name', 'artists', ) + exporter = TableExport( + export_format=export_format, + table=self.get_table(exclude=export_exclude), + exclude_columns=self.exclude_columns, + ) + return exporter.response(filename=self.get_export_filename(export_format)) diff --git a/requirements.txt b/requirements.txt index 936771a..23befcc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,6 @@ python-dateutil==2.7.5 pytz==2018.4 requests==2.18.4 six==1.11.0 +tablib==0.12.1 urllib3==1.22 wrapt==1.10.11 From d15717490d5718259c3c0b5fc9295940a7b03c74 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Tue, 6 Nov 2018 23:23:44 -0500 Subject: [PATCH 11/20] Export ISO timestamp for history (#57) --- api/models.py | 9 +++------ graphs/utils.py | 5 +++-- graphs/views.py | 5 +++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/api/models.py b/api/models.py index be099d0..5d6b4cf 100644 --- a/api/models.py +++ b/api/models.py @@ -103,15 +103,12 @@ class History(models.Model): def __str__(self): return " - ".join((str(self.user), str(self.timestamp), str(self.track))) - def get_track_name(self): - return self.track.name - - def get_track_id(self): - return self.track.id - def get_artists(self): artist_names = [artist.name for artist in self.track.artists.all()] return ', '.join(artist_names) + def get_iso_timestamp(self): + return self.timestamp.isoformat() + # }}} # diff --git a/graphs/utils.py b/graphs/utils.py index 69e262d..25be3b6 100644 --- a/graphs/utils.py +++ b/graphs/utils.py @@ -9,8 +9,9 @@ class HistoryTable(tables.Table): model = History template_name = 'django_tables2/bootstrap.html' - track_name = tables.Column(accessor='get_track_name', orderable=False) - track_id = tables.Column(accessor='get_track_id', orderable=False) + iso_timestamp = tables.Column(accessor='get_iso_timestamp', orderable=False) + track_name = tables.Column(accessor='track.name', orderable=False) + track_id = tables.Column(accessor='track.id', orderable=False) artists = tables.Column(accessor='get_artists', orderable=False) def get_secret_context(user_secret): diff --git a/graphs/views.py b/graphs/views.py index 37b1c5c..ff0bea5 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -53,7 +53,7 @@ class HistoryList(ExportMixin, SingleTableView): template_name = 'graphs/user_history.html' def get_table_kwargs(self): - return { 'exclude': ('id', 'user', 'track', 'track_id', ) } + return { 'exclude': ('id', 'user', 'track', 'track_id', 'iso_timestamp', ) } def get_table_data(self): return History.objects.filter(user__exact=self.request.session['user_id']).order_by('-timestamp') @@ -67,7 +67,8 @@ class HistoryList(ExportMixin, SingleTableView): return "{}.{}".format(self.request.session['user_id'], export_format) def create_export(self, export_format): - export_exclude = ('id', 'user', 'track', 'track_name', 'artists', ) + export_exclude = ('id', 'user', 'track', 'track_name', 'artists', + 'timestamp', ) exporter = TableExport( export_format=export_format, table=self.get_table(exclude=export_exclude), From df62fc21eee51eb808ef3a80700fe0ba46053614 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 7 Nov 2018 00:14:45 -0500 Subject: [PATCH 12/20] Copied Chris' code to get genre graph working --- api/views.py | 4 ++-- graphs/templates/graphs/genre_graph.html | 11 +---------- graphs/urls.py | 3 --- graphs/utils.py | 2 +- 4 files changed, 4 insertions(+), 16 deletions(-) diff --git a/api/views.py b/api/views.py index 741c4c3..6dc204d 100644 --- a/api/views.py +++ b/api/views.py @@ -211,7 +211,7 @@ def get_artist_data(request, user_secret): filter=Q(track__users=user))) processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts] - pprint.pprint(processed_artist_counts) + pprint(processed_artist_counts) return JsonResponse(data=processed_artist_counts, safe=False) # }}} get_artist_data # @@ -257,7 +257,7 @@ def get_genre_data(request, user_secret): genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre'], genre_dict['num_songs']) print("*** Genre Breakdown ***") - pprint.pprint(list(genre_counts)) + pprint(list(genre_counts)) return JsonResponse(data=list(genre_counts), safe=False) # }}} get_genre_data # diff --git a/graphs/templates/graphs/genre_graph.html b/graphs/templates/graphs/genre_graph.html index 4eb5940..7d35ee8 100644 --- a/graphs/templates/graphs/genre_graph.html +++ b/graphs/templates/graphs/genre_graph.html @@ -46,16 +46,7 @@ var y = d3.scaleLinear() .rangeRound([height, 0]); - // PU: trying to store genreData locally - var genreData; - d3.json("{% url "api:get_genre_data" user_secret %}", - function(data) { - genreData = data; - } - ); - console.log(genreData); - create_genre_graph(genreData); - // d3.json("{% url "api:get_genre_data" user_secret %}").then(create_genre_graph); + d3.json("{% url "api:get_genre_data" user_secret %}").then(create_genre_graph); diff --git a/graphs/urls.py b/graphs/urls.py index 6ff4da5..527822b 100644 --- a/graphs/urls.py +++ b/graphs/urls.py @@ -10,8 +10,5 @@ urlpatterns = [ name='display_genre_graph'), path('audio_features/', display_features_graphs, name='display_audio_features'), - # path('history/', display_history_table, - # name='display_history_table'), - # path('history/', display_history_table, name='display_history_table'), path('history/', HistoryList.as_view(), name='display_history_table'), ] diff --git a/graphs/utils.py b/graphs/utils.py index 25be3b6..14c037e 100644 --- a/graphs/utils.py +++ b/graphs/utils.py @@ -10,8 +10,8 @@ class HistoryTable(tables.Table): template_name = 'django_tables2/bootstrap.html' iso_timestamp = tables.Column(accessor='get_iso_timestamp', orderable=False) - track_name = tables.Column(accessor='track.name', orderable=False) track_id = tables.Column(accessor='track.id', orderable=False) + track_name = tables.Column(accessor='track.name', orderable=False) artists = tables.Column(accessor='get_artists', orderable=False) def get_secret_context(user_secret): From a4a00458af601f4de171dabd57c065c18a20e613 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 7 Nov 2018 17:48:11 -0500 Subject: [PATCH 13/20] Able to upload history onto server (#57) --- .gitignore | 1 + graphs/templates/graphs/user_history.html | 2 +- graphs/views.py | 9 +++++++++ login/forms.py | 8 ++++++++ login/models.py | 5 +++++ login/templates/login/scan.html | 5 +++++ login/urls.py | 1 + login/views.py | 21 ++++++++++++++++++++- spotifyvis/settings.py | 3 +++ 9 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 login/forms.py diff --git a/.gitignore b/.gitignore index c492bbd..73bb973 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ api-keys.sh Pipfile *.txt scrap.py +media/history/* diff --git a/graphs/templates/graphs/user_history.html b/graphs/templates/graphs/user_history.html index 8d41a06..39c2e27 100644 --- a/graphs/templates/graphs/user_history.html +++ b/graphs/templates/graphs/user_history.html @@ -10,7 +10,7 @@

{{ user_id }}'s Listening History

- Export CSV + Export {% render_table user_history_table %} diff --git a/graphs/views.py b/graphs/views.py index ff0bea5..404eaac 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -45,6 +45,8 @@ def display_features_graphs(request, user_secret): return render(request, "graphs/features_graphs.html", get_secret_context(user_secret)) +# HistoryList {{{ # + class HistoryList(ExportMixin, SingleTableView): """Create table with list of song history.""" model = History @@ -52,6 +54,8 @@ class HistoryList(ExportMixin, SingleTableView): context_table_name = 'user_history_table' template_name = 'graphs/user_history.html' + # overridden methods {{{ # + def get_table_kwargs(self): return { 'exclude': ('id', 'user', 'track', 'track_id', 'iso_timestamp', ) } @@ -76,3 +80,8 @@ class HistoryList(ExportMixin, SingleTableView): ) return exporter.response(filename=self.get_export_filename(export_format)) + + # }}} overridden methods # + +# }}} HistoryList # + diff --git a/login/forms.py b/login/forms.py new file mode 100644 index 0000000..3a70ea8 --- /dev/null +++ b/login/forms.py @@ -0,0 +1,8 @@ +from django import forms +from .models import HistoryUpload + +class HistoryUploadForm(forms.ModelForm): + class Meta: + model = HistoryUpload + fields = ('user_id', 'document', ) + # widgets = { 'user_id': forms.HiddenInput() } diff --git a/login/models.py b/login/models.py index aa04baf..1bc0474 100644 --- a/login/models.py +++ b/login/models.py @@ -20,3 +20,8 @@ class User(models.Model): def __str__(self): return self.id + +class HistoryUpload(models.Model): + user_id = models.ForeignKey(User, on_delete=models.CASCADE) + document = models.FileField(upload_to='history/') + uploaded_at = models.DateTimeField(auto_now_add=True) diff --git a/login/templates/login/scan.html b/login/templates/login/scan.html index e0c6dce..7ea75a4 100644 --- a/login/templates/login/scan.html +++ b/login/templates/login/scan.html @@ -24,5 +24,10 @@ Scan History +
+ {% csrf_token %} + {{ form.as_p }} + +
diff --git a/login/urls.py b/login/urls.py index 75c29e8..120cf7f 100644 --- a/login/urls.py +++ b/login/urls.py @@ -9,4 +9,5 @@ urlpatterns = [ path('callback', callback, name='callback'), # path('user/', user_home, name='user_home'), path('admin_graphs', admin_graphs, name='admin_graphs'), + path('upload_history', upload_history, name='upload_history'), ] diff --git a/login/views.py b/login/views.py index 649ced2..8872993 100644 --- a/login/views.py +++ b/login/views.py @@ -11,6 +11,7 @@ from django.shortcuts import render, redirect from django.http import HttpResponseBadRequest from .models import * from .utils import * +from .forms import HistoryUploadForm # }}} imports # @@ -78,7 +79,10 @@ def callback(request): request.session['user_id'] = user_obj.id request.session['user_secret'] = user_obj.secret - return render(request, 'login/scan.html', get_user_context(user_obj)) + context = get_user_context(user_obj) + context['form'] = HistoryUploadForm() + + return render(request, 'login/scan.html', context) # }}} callback # @@ -97,3 +101,18 @@ def admin_graphs(request): return render(request, 'graphs/logged_in.html', get_user_context(user_obj)) # }}} admin_graphs # + +def upload_history(request): + if request.method == 'POST': + form = HistoryUploadForm(request.POST, request.FILES) + form.fields['user_id'].initial = User.objects.get(id=request.session['user_id']) + if form.is_valid(): + form.save() + + # Redirect to the document list after POST + return redirect('graphs:display_history_table') + else: + form = HistoryUploadForm() + + # return redirect('graphs:display_history_table') + return render(request, 'login/scan.html', context) diff --git a/spotifyvis/settings.py b/spotifyvis/settings.py index b8db2ef..4c85115 100644 --- a/spotifyvis/settings.py +++ b/spotifyvis/settings.py @@ -130,3 +130,6 @@ STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(BASE_DIR, "static"), ] + +MEDIA_URL = '/media/' +MEDIA_ROOT = os.path.join(BASE_DIR, 'media') From 93e646565a1e34907b36821d53b207dd84c497e1 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 7 Nov 2018 18:07:23 -0500 Subject: [PATCH 14/20] Set uploader to current user in HistoryUpload (#57) --- login/forms.py | 4 ++-- login/models.py | 2 +- login/utils.py | 14 +++++++++++++- login/views.py | 13 ++----------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/login/forms.py b/login/forms.py index 3a70ea8..3863e50 100644 --- a/login/forms.py +++ b/login/forms.py @@ -4,5 +4,5 @@ from .models import HistoryUpload class HistoryUploadForm(forms.ModelForm): class Meta: model = HistoryUpload - fields = ('user_id', 'document', ) - # widgets = { 'user_id': forms.HiddenInput() } + fields = ('user', 'document', ) + widgets = { 'user': forms.HiddenInput() } diff --git a/login/models.py b/login/models.py index 1bc0474..2679b61 100644 --- a/login/models.py +++ b/login/models.py @@ -22,6 +22,6 @@ class User(models.Model): return self.id class HistoryUpload(models.Model): - user_id = models.ForeignKey(User, on_delete=models.CASCADE) + user = models.ForeignKey(User, on_delete=models.CASCADE) document = models.FileField(upload_to='history/') uploaded_at = models.DateTimeField(auto_now_add=True) diff --git a/login/utils.py b/login/utils.py index 1f2bdef..02f7e7e 100644 --- a/login/utils.py +++ b/login/utils.py @@ -13,7 +13,6 @@ def get_user_context(user_obj): """ return { 'user_id': user_obj.id, 'user_secret': user_obj.secret, } - # generate_random_string {{{ # def generate_random_string(length): @@ -67,3 +66,16 @@ def create_user(refresh_token, access_token, access_expires_in): # }}} create_user # +def get_scan_context(request): + """Get context for rendering scan page. + + :request: + :returns: Context with upload form and user info. + + """ + context = { 'user_id': request.session['user_id'], + 'user_secret': request.session['user_secret'], } + # set hidden user field to current user + context['form'] = HistoryUploadForm(initial= + { 'user': User.objects.get(id=request.session['user_id']) }) + return context diff --git a/login/views.py b/login/views.py index 8872993..7707dfd 100644 --- a/login/views.py +++ b/login/views.py @@ -79,10 +79,7 @@ def callback(request): request.session['user_id'] = user_obj.id request.session['user_secret'] = user_obj.secret - context = get_user_context(user_obj) - context['form'] = HistoryUploadForm() - - return render(request, 'login/scan.html', context) + return render(request, 'login/scan.html', get_scan_context(request)) # }}} callback # @@ -105,14 +102,8 @@ def admin_graphs(request): def upload_history(request): if request.method == 'POST': form = HistoryUploadForm(request.POST, request.FILES) - form.fields['user_id'].initial = User.objects.get(id=request.session['user_id']) if form.is_valid(): form.save() - - # Redirect to the document list after POST return redirect('graphs:display_history_table') - else: - form = HistoryUploadForm() - # return redirect('graphs:display_history_table') - return render(request, 'login/scan.html', context) + return render(request, 'login/scan.html', get_scan_context(request)) From 920a9ad7724c1c3348a56757c6d4ab2030a14add Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 7 Nov 2018 21:54:14 -0500 Subject: [PATCH 15/20] Import history into DB from exported CSV (#57) --- api/urls.py | 2 + api/views.py | 103 +++++++++++++++++++++++++++++++++++++++++++++++++ login/utils.py | 2 + login/views.py | 7 ++-- 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/api/urls.py b/api/urls.py index 2d9d78b..80ad625 100644 --- a/api/urls.py +++ b/api/urls.py @@ -14,4 +14,6 @@ urlpatterns = [ name='get_genre_data'), path('audio_features//', get_audio_feature_data, name='get_audio_feature_data'), + path('import/history/', import_history, name='import_history'), ] + diff --git a/api/views.py b/api/views.py index 6dc204d..f29b7c1 100644 --- a/api/views.py +++ b/api/views.py @@ -6,22 +6,26 @@ import requests import urllib import secrets import string +import csv from django.shortcuts import render, redirect from django.http import JsonResponse from django.db.models import Count, Q, Max +from django.core.files import File from .utils import * from .models import * from login.models import User from login.utils import get_user_context from dateutil.parser import parse from pprint import pprint +from login.models import HistoryUpload # }}} imports # # constants {{{ # USER_TRACKS_LIMIT = 50 +TRACKS_LIMIT = 50 HISTORY_LIMIT = 50 ARTIST_LIMIT = 50 FEATURES_LIMIT = 100 @@ -29,6 +33,7 @@ FEATURES_LIMIT = 100 # FEATURES_LIMIT = 25 TRACKS_TO_QUERY = 100 HISTORY_ENDPOINT = 'https://api.spotify.com/v1/me/player/recently-played' +TRACKS_ENDPOINT = 'https://api.spotify.com/v1/tracks' console_logging = True # console_logging = False @@ -261,3 +266,101 @@ def get_genre_data(request, user_secret): return JsonResponse(data=list(genre_counts), safe=False) # }}} get_genre_data # + +# import_history {{{ # + +def import_history(request, upload_id): + """Import history for the user from the file they uploaded. + + :upload_id: ID (PK) of the HistoryUpload entry + :returns: None + """ + + headers = ['timestamp', 'track_id'] + upload_obj = HistoryUpload.objects.get(id=upload_id) + user_headers = get_user_header(upload_obj.user) + + with upload_obj.document.open('r') as f: + csv_reader = csv.reader(f, delimiter=',') + rows_read = 0 + history_obj_info_lst = [] + artist_genre_queue = [] + + next(csv_reader) + row = next(csv_reader) + last_row = False + while not last_row: + # if Track.objects.filter(id__exact=row[1]).exists(): + history_obj_info = {} + for i in range(len(headers)): + history_obj_info[headers[i]] = row[i] + try: + row = next(csv_reader) + except StopIteration: + last_row = True + history_obj_info_lst.append(history_obj_info) + rows_read += 1 + if (rows_read % TRACKS_LIMIT == 0) or last_row: + track_ids_lst = [info['track_id'] for info in history_obj_info_lst] + # print(len(track_ids_lst)) + track_ids = ','.join(track_ids_lst) + payload = {'ids': track_ids} + tracks_response = requests.get(TRACKS_ENDPOINT, + headers=user_headers, + params=payload).json()['tracks'] + responses_processed = 0 + + for track_dict in tracks_response: + # add artists {{{ # + + # update artist info before track so that Track object can reference + # Artist object + track_artists = [] + for artist_dict in track_dict['artists']: + artist_obj, artist_created = Artist.objects.get_or_create( + id=artist_dict['id'], + name=artist_dict['name'],) + # only add/tally up artist genres if new + if artist_created: + artist_genre_queue.append(artist_obj) + if len(artist_genre_queue) == ARTIST_LIMIT: + add_artist_genres(user_headers, artist_genre_queue) + artist_genre_queue = [] + track_artists.append(artist_obj) + + # }}} add artists # + + # don't associate history track with User, not necessarily in their + # library + track_obj, track_created = save_track_obj(track_dict, track_artists, None) + + timestamp = parse(history_obj_info_lst[responses_processed]['timestamp']) + # missing PK to do get_or_create + history_query = \ + History.objects.filter(user__exact=upload_obj.user, + timestamp__exact=timestamp) + if len(history_query) == 0: + history_obj = \ + History.objects.create(user=upload_obj.user, + timestamp=timestamp, + track=track_obj,) + else: + history_obj = history_query[0] + + if console_logging: + print("Processed row #{}: {}".format( + (rows_read - TRACKS_LIMIT) + responses_processed, history_obj,)) + responses_processed += 1 + + history_obj_info_lst = [] + + if len(artist_genre_queue) > 0: + add_artist_genres(user_headers, artist_genre_queue) + + # TODO: update track genres from History relation + # update_track_genres(user_obj) + + return redirect('graphs:display_history_table') + +# }}} get_history # + diff --git a/login/utils.py b/login/utils.py index 02f7e7e..d67ede8 100644 --- a/login/utils.py +++ b/login/utils.py @@ -1,8 +1,10 @@ import string import random import requests +import secrets from .models import User +from .forms import HistoryUploadForm def get_user_context(user_obj): """Get context for rendering with User's ID and secret. diff --git a/login/views.py b/login/views.py index 7707dfd..0479fa2 100644 --- a/login/views.py +++ b/login/views.py @@ -3,7 +3,6 @@ import math import os import urllib -import secrets import pprint from datetime import datetime @@ -11,7 +10,6 @@ from django.shortcuts import render, redirect from django.http import HttpResponseBadRequest from .models import * from .utils import * -from .forms import HistoryUploadForm # }}} imports # @@ -103,7 +101,8 @@ def upload_history(request): if request.method == 'POST': form = HistoryUploadForm(request.POST, request.FILES) if form.is_valid(): - form.save() - return redirect('graphs:display_history_table') + upload_obj = form.save() + # return redirect('graphs:display_history_table') + return redirect('api:import_history', upload_id=upload_obj.id) return render(request, 'login/scan.html', get_scan_context(request)) From f54eb4a81471893952e6dde5dcb8bdbc46bb872f Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 7 Nov 2018 22:50:32 -0500 Subject: [PATCH 16/20] Refactor saving artists to track Exact same code in parse_history, parse_library and import_history. --- api/utils.py | 27 +++++++++++++---- api/views.py | 86 ++++++++++++++++------------------------------------ 2 files changed, 47 insertions(+), 66 deletions(-) diff --git a/api/utils.py b/api/utils.py index c6a384c..0f8ec8b 100644 --- a/api/utils.py +++ b/api/utils.py @@ -1,7 +1,6 @@ # imports {{{ # import requests import math -import pprint import os import json @@ -10,7 +9,9 @@ from django.http import JsonResponse from django.core import serializers from django.utils import timezone from .models import * +from . import views from login.models import User +from pprint import pprint # }}} imports # @@ -182,7 +183,7 @@ def add_artist_genres(headers, artist_objs): artist_ids = str.join(",", [artist_obj.id for artist_obj in artist_objs]) params = {'ids': artist_ids} artists_response = requests.get('https://api.spotify.com/v1/artists/', - headers=headers, + headers=headers, params={'ids': artist_ids}, ).json()['artists'] for i in range(len(artist_objs)): @@ -235,14 +236,28 @@ def get_artists_in_genre(user, genre, max_songs): # }}} get_artists_in_genre # -def create_artist_for_track(artist_dict): - """TODO: Docstring for create_artist_for_track. +def save_track_artists(track_dict, artist_genre_queue, user_headers): + """ Update artist info before creating Track so that Track object can + reference Artist object. - :artist_dict: TODO + :track_dict: TODO :returns: None """ - pass + track_artists = [] + for artist_dict in track_dict['artists']: + artist_obj, artist_created = Artist.objects.get_or_create( + id=artist_dict['id'], + name=artist_dict['name'],) + # only add/tally up artist genres if new + if artist_created: + artist_genre_queue.append(artist_obj) + if len(artist_genre_queue) == views.ARTIST_LIMIT: + add_artist_genres(user_headers, artist_genre_queue) + artist_genre_queue[:] = [] + track_artists.append(artist_obj) + + return track_artists def get_user_header(user_obj): """Returns the authorization string needed to make an API call. diff --git a/api/views.py b/api/views.py index f29b7c1..a304ebb 100644 --- a/api/views.py +++ b/api/views.py @@ -60,7 +60,8 @@ def parse_library(request, user_secret): # create this obj so loop runs at least once saved_tracks_response = [0] # scan until reach num_tracks or no tracks left if scanning entire library - while (TRACKS_TO_QUERY == 0 or offset < TRACKS_TO_QUERY) and len(saved_tracks_response) > 0: + while ((TRACKS_TO_QUERY == 0 or offset < TRACKS_TO_QUERY) and + len(saved_tracks_response) > 0): payload['offset'] = str(offset) saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', headers=user_headers, @@ -70,25 +71,8 @@ def parse_library(request, user_secret): tracks_processed = 0 for track_dict in saved_tracks_response: - # add artists {{{ # - - # update artist info before track so that Track object can reference - # Artist object - track_artists = [] - for artist_dict in track_dict['track']['artists']: - artist_obj, artist_created = Artist.objects.get_or_create( - id=artist_dict['id'], - name=artist_dict['name'],) - # only add/tally up artist genres if new - if artist_created: - artist_genre_queue.append(artist_obj) - if len(artist_genre_queue) == ARTIST_LIMIT: - add_artist_genres(user_headers, artist_genre_queue) - artist_genre_queue = [] - track_artists.append(artist_obj) - - # }}} add artists # - + track_artists = save_track_artists(track_dict['track'], artist_genre_queue, + user_headers) track_obj, track_created = save_track_obj(track_dict['track'], track_artists, user_obj) @@ -158,29 +142,14 @@ def parse_history(request, user_secret): tracks_processed = 0 for track_dict in history_response: - # add artists {{{ # - - # update artist info before track so that Track object can reference - # Artist object - track_artists = [] - for artist_dict in track_dict['track']['artists']: - artist_obj, artist_created = Artist.objects.get_or_create( - id=artist_dict['id'], - name=artist_dict['name'],) - # only add/tally up artist genres if new - if artist_created: - artist_genre_queue.append(artist_obj) - if len(artist_genre_queue) == ARTIST_LIMIT: - add_artist_genres(user_headers, artist_genre_queue) - artist_genre_queue = [] - track_artists.append(artist_obj) - - # }}} add artists # - # don't associate history track with User, not necessarily in their # library - track_obj, track_created = save_track_obj(track_dict['track'], - track_artists, None) + # track_obj, track_created = save_track_obj(track_dict['track'], + # track_artists, None) + track_artists = save_track_artists(track_dict['track'], artist_genre_queue, + user_headers) + track_obj, track_created = save_track_obj(track_dict['track'], + track_artists, None) history_obj, history_created = History.objects.get_or_create( user=user_obj, timestamp=parse(track_dict['played_at']), @@ -276,6 +245,8 @@ def import_history(request, upload_id): :returns: None """ + # setup {{{ # + headers = ['timestamp', 'track_id'] upload_obj = HistoryUpload.objects.get(id=upload_id) user_headers = get_user_header(upload_obj.user) @@ -290,6 +261,7 @@ def import_history(request, upload_id): row = next(csv_reader) last_row = False while not last_row: + # if Track.objects.filter(id__exact=row[1]).exists(): history_obj_info = {} for i in range(len(headers)): @@ -298,9 +270,14 @@ def import_history(request, upload_id): row = next(csv_reader) except StopIteration: last_row = True + + # }}} setup # + history_obj_info_lst.append(history_obj_info) rows_read += 1 if (rows_read % TRACKS_LIMIT == 0) or last_row: + # get tracks_response {{{ # + track_ids_lst = [info['track_id'] for info in history_obj_info_lst] # print(len(track_ids_lst)) track_ids = ','.join(track_ids_lst) @@ -309,29 +286,16 @@ def import_history(request, upload_id): headers=user_headers, params=payload).json()['tracks'] responses_processed = 0 + + # }}} get tracks_response # for track_dict in tracks_response: - # add artists {{{ # - - # update artist info before track so that Track object can reference - # Artist object - track_artists = [] - for artist_dict in track_dict['artists']: - artist_obj, artist_created = Artist.objects.get_or_create( - id=artist_dict['id'], - name=artist_dict['name'],) - # only add/tally up artist genres if new - if artist_created: - artist_genre_queue.append(artist_obj) - if len(artist_genre_queue) == ARTIST_LIMIT: - add_artist_genres(user_headers, artist_genre_queue) - artist_genre_queue = [] - track_artists.append(artist_obj) - - # }}} add artists # - + # create History obj {{{ # + # don't associate history track with User, not necessarily in their # library + track_artists = save_track_artists(track_dict, artist_genre_queue, + user_headers) track_obj, track_created = save_track_obj(track_dict, track_artists, None) timestamp = parse(history_obj_info_lst[responses_processed]['timestamp']) @@ -351,6 +315,8 @@ def import_history(request, upload_id): print("Processed row #{}: {}".format( (rows_read - TRACKS_LIMIT) + responses_processed, history_obj,)) responses_processed += 1 + + # }}} create History obj # history_obj_info_lst = [] From 3d0acc7a4b3c55a91fd67be8f395d74da3de457a Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 8 Nov 2018 09:39:10 -0500 Subject: [PATCH 17/20] Refactor saving History obj From import_history and parse_history. Also refactor getting CSV information from import_history. --- api/utils.py | 51 ++++++++++++++++++++++- api/views.py | 48 +++++++-------------- graphs/templates/graphs/user_history.html | 1 + graphs/views.py | 1 + 4 files changed, 65 insertions(+), 36 deletions(-) diff --git a/api/utils.py b/api/utils.py index 0f8ec8b..c97b7d4 100644 --- a/api/utils.py +++ b/api/utils.py @@ -236,12 +236,14 @@ def get_artists_in_genre(user, genre, max_songs): # }}} get_artists_in_genre # +# save_track_artists {{{ # + def save_track_artists(track_dict, artist_genre_queue, user_headers): """ Update artist info before creating Track so that Track object can reference Artist object. - :track_dict: TODO - :returns: None + :track_dict: response from Spotify API for track + :returns: list of Artist objects in Track """ track_artists = [] @@ -259,6 +261,10 @@ def save_track_artists(track_dict, artist_genre_queue, user_headers): return track_artists +# }}} save_track_artists # + +# get_user_header {{{ # + def get_user_header(user_obj): """Returns the authorization string needed to make an API call. @@ -284,3 +290,44 @@ def get_user_header(user_obj): user_obj.save() return {'Authorization': "Bearer " + user_obj.access_token} + +# }}} get_user_header # + +def save_history_obj (user, timestamp, track): + """Return (get/create) a History object with the specified parameters. Can't + use built-in get_or_create since don't know auto PK. + + :user: User object History should be associated with + :timestamp: time at which song was listened to + :track: Track object for song + :returns: History object + + """ + history_query = History.objects.filter(user__exact=user, + timestamp__exact=timestamp) + if len(history_query) == 0: + history_obj = History.objects.create(user=user, timestamp=timestamp, + track=track) + else: + history_obj = history_query[0] + + return history_obj + +def get_next_history_row(csv_reader, headers, prev_info): + """Return formatted information from next row in history CSV file. + + :csv_reader: TODO + :headers: + :prev_info: history_obj_info of last row in case no more rows + :returns: (boolean of if last row, dict with information of next row) + + """ + try: + row = next(csv_reader) + # if Track.objects.filter(id__exact=row[1]).exists(): + history_obj_info = {} + for i in range(len(headers)): + history_obj_info[headers[i]] = row[i] + return False, history_obj_info + except StopIteration: + return True, prev_info diff --git a/api/views.py b/api/views.py index a304ebb..a792124 100644 --- a/api/views.py +++ b/api/views.py @@ -150,10 +150,8 @@ def parse_history(request, user_secret): user_headers) track_obj, track_created = save_track_obj(track_dict['track'], track_artists, None) - history_obj, history_created = History.objects.get_or_create( - user=user_obj, - timestamp=parse(track_dict['played_at']), - track=track_obj,) + history_obj = save_history_obj(user_obj, parse(track_dict['played_at']), + track_obj) if console_logging: tracks_processed += 1 @@ -257,19 +255,12 @@ def import_history(request, upload_id): history_obj_info_lst = [] artist_genre_queue = [] - next(csv_reader) - row = next(csv_reader) - last_row = False + # skip header row + last_row, history_obj_info = get_next_history_row(csv_reader, headers, + {}) while not last_row: - - # if Track.objects.filter(id__exact=row[1]).exists(): - history_obj_info = {} - for i in range(len(headers)): - history_obj_info[headers[i]] = row[i] - try: - row = next(csv_reader) - except StopIteration: - last_row = True + last_row, history_obj_info = get_next_history_row(csv_reader, + headers, history_obj_info) # }}} setup # @@ -290,33 +281,22 @@ def import_history(request, upload_id): # }}} get tracks_response # for track_dict in tracks_response: - # create History obj {{{ # - # don't associate history track with User, not necessarily in their # library track_artists = save_track_artists(track_dict, artist_genre_queue, user_headers) - track_obj, track_created = save_track_obj(track_dict, track_artists, None) - - timestamp = parse(history_obj_info_lst[responses_processed]['timestamp']) - # missing PK to do get_or_create - history_query = \ - History.objects.filter(user__exact=upload_obj.user, - timestamp__exact=timestamp) - if len(history_query) == 0: - history_obj = \ - History.objects.create(user=upload_obj.user, - timestamp=timestamp, - track=track_obj,) - else: - history_obj = history_query[0] + track_obj, track_created = save_track_obj(track_dict, + track_artists, None) + + timestamp = \ + parse(history_obj_info_lst[responses_processed]['timestamp']) + history_obj = save_history_obj(upload_obj.user, timestamp, + track_obj) if console_logging: print("Processed row #{}: {}".format( (rows_read - TRACKS_LIMIT) + responses_processed, history_obj,)) responses_processed += 1 - - # }}} create History obj # history_obj_info_lst = [] diff --git a/graphs/templates/graphs/user_history.html b/graphs/templates/graphs/user_history.html index 39c2e27..a7cc84b 100644 --- a/graphs/templates/graphs/user_history.html +++ b/graphs/templates/graphs/user_history.html @@ -10,6 +10,7 @@

{{ user_id }}'s Listening History

+

Found {{ total_history }} songs.

Export {% render_table user_history_table %} diff --git a/graphs/views.py b/graphs/views.py index 404eaac..2c09239 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -65,6 +65,7 @@ class HistoryList(ExportMixin, SingleTableView): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) context['user_id'] = self.request.session['user_id'] + context['total_history'] = self.get_table_data().count() return context def get_export_filename(self, export_format): From db29bc9f67101295fcdc75d734ff1e42f92dadf5 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Wed, 14 Nov 2018 12:02:46 -0500 Subject: [PATCH 18/20] Added timestamp to exported history filename (#57) --- api/views.py | 3 +++ graphs/views.py | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/api/views.py b/api/views.py index a792124..be455a0 100644 --- a/api/views.py +++ b/api/views.py @@ -265,6 +265,9 @@ def import_history(request, upload_id): # }}} setup # history_obj_info_lst.append(history_obj_info) + # PU: refactor saving History object right away if Track obj already + # exists + # PU: refactor below? rows_read += 1 if (rows_read % TRACKS_LIMIT == 0) or last_row: # get tracks_response {{{ # diff --git a/graphs/views.py b/graphs/views.py index 2c09239..8c410d6 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -6,10 +6,11 @@ import requests import os import urllib import secrets -from pprint import pprint import string -from datetime import datetime +from pprint import pprint +from datetime import datetime +from time import strftime from django.shortcuts import render, redirect from .utils import * from django_tables2 import RequestConfig, SingleTableView @@ -69,7 +70,9 @@ class HistoryList(ExportMixin, SingleTableView): return context def get_export_filename(self, export_format): - return "{}.{}".format(self.request.session['user_id'], export_format) + user_id = self.request.session['user_id'] + timestamp = strftime("%m%d%Y-%H%M") + return "{}.{}".format("-".join((user_id, timestamp)), export_format) def create_export(self, export_format): export_exclude = ('id', 'user', 'track', 'track_name', 'artists', From 895cf7d40d06e3fef67108ec543198567de10d3a Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Thu, 15 Nov 2018 01:23:50 -0500 Subject: [PATCH 19/20] Split parse_history into own function For grabbing history in a command (#58). --- api/urls.py | 2 +- api/utils.py | 68 ++++++++++++++++++++++++++++++++++-- api/views.py | 98 ++++++++++++++++++++++++++++++---------------------- 3 files changed, 122 insertions(+), 46 deletions(-) diff --git a/api/urls.py b/api/urls.py index 80ad625..4769690 100644 --- a/api/urls.py +++ b/api/urls.py @@ -6,7 +6,7 @@ app_name = 'api' urlpatterns = [ path('scan/library/', parse_library, name='scan_library'), - path('scan/history/', parse_history, + path('scan/history/', parse_history_request, name='scan_history'), path('user_artists/', get_artist_data, name='get_artist_data'), diff --git a/api/utils.py b/api/utils.py index c97b7d4..9f24864 100644 --- a/api/utils.py +++ b/api/utils.py @@ -4,7 +4,7 @@ import math import os import json -from django.db.models import Count, Q, F +from django.db.models import Count, F, Max from django.http import JsonResponse from django.core import serializers from django.utils import timezone @@ -12,11 +12,14 @@ from .models import * from . import views from login.models import User from pprint import pprint +from dateutil.parser import parse + +HISTORY_ENDPOINT = 'https://api.spotify.com/v1/me/player/recently-played' # }}} imports # -# console_logging = True -console_logging = False +console_logging = True +# console_logging = False artists_genre_processed = 0 features_processed = 0 @@ -293,6 +296,8 @@ def get_user_header(user_obj): # }}} get_user_header # +# save_history_obj {{{ # + def save_history_obj (user, timestamp, track): """Return (get/create) a History object with the specified parameters. Can't use built-in get_or_create since don't know auto PK. @@ -313,6 +318,10 @@ def save_history_obj (user, timestamp, track): return history_obj +# }}} save_history_obj # + +# get_next_history_row {{{ # + def get_next_history_row(csv_reader, headers, prev_info): """Return formatted information from next row in history CSV file. @@ -331,3 +340,56 @@ def get_next_history_row(csv_reader, headers, prev_info): return False, history_obj_info except StopIteration: return True, prev_info + +# }}} get_next_history_row # + +# parse_history {{{ # + +def parse_history(user_secret): + """Scans user's listening history and stores the information in a + database. + + :user_secret: secret for User object who's library is being scanned. + :returns: None + """ + + user_obj = User.objects.get(secret=user_secret) + payload = {'limit': str(views.USER_TRACKS_LIMIT)} + last_time_played = History.objects.filter(user=user_obj).aggregate(Max('timestamp'))['timestamp__max'] + if last_time_played is not None: + payload['after'] = last_time_played.isoformat() + artist_genre_queue = [] + user_headers = get_user_header(user_obj) + history_response = requests.get(HISTORY_ENDPOINT, + headers=user_headers, + params=payload).json()['items'] + # pprint(history_response) + + if console_logging: + tracks_processed = 0 + + for track_dict in history_response: + # don't associate history track with User, not necessarily in their + # library + # track_obj, track_created = save_track_obj(track_dict['track'], + # track_artists, None) + track_artists = save_track_artists(track_dict['track'], artist_genre_queue, + user_headers) + track_obj, track_created = save_track_obj(track_dict['track'], + track_artists, None) + history_obj = save_history_obj(user_obj, parse(track_dict['played_at']), + track_obj) + + if console_logging: + tracks_processed += 1 + print("Added history track #{}: {}".format( + tracks_processed, history_obj,)) + + if len(artist_genre_queue) > 0: + add_artist_genres(user_headers, artist_genre_queue) + + # TODO: update track genres from History relation + # update_track_genres(user_obj) + +# }}} get_history # + diff --git a/api/views.py b/api/views.py index be455a0..bd6df30 100644 --- a/api/views.py +++ b/api/views.py @@ -32,7 +32,6 @@ FEATURES_LIMIT = 100 # ARTIST_LIMIT = 25 # FEATURES_LIMIT = 25 TRACKS_TO_QUERY = 100 -HISTORY_ENDPOINT = 'https://api.spotify.com/v1/me/player/recently-played' TRACKS_ENDPOINT = 'https://api.spotify.com/v1/tracks' console_logging = True @@ -116,55 +115,70 @@ def parse_library(request, user_secret): # }}} parse_library # -# parse_history {{{ # +# # parse_history {{{ # + +# def parse_history(request, user_secret): + # """Scans user's listening history and stores the information in a + # database. + + # :user_secret: secret for User object who's library is being scanned. + # :returns: None + # """ + + # user_obj = User.objects.get(secret=user_secret) + # payload = {'limit': str(USER_TRACKS_LIMIT)} + # last_time_played = History.objects.filter(user=user_obj).aggregate(Max('timestamp'))['timestamp__max'] + # if last_time_played is not None: + # payload['after'] = last_time_played.isoformat() + # artist_genre_queue = [] + # user_headers = get_user_header(user_obj) + # history_response = requests.get(HISTORY_ENDPOINT, + # headers=user_headers, + # params=payload).json()['items'] + # # pprint(history_response) + + # if console_logging: + # tracks_processed = 0 + + # for track_dict in history_response: + # # don't associate history track with User, not necessarily in their + # # library + # # track_obj, track_created = save_track_obj(track_dict['track'], + # # track_artists, None) + # track_artists = save_track_artists(track_dict['track'], artist_genre_queue, + # user_headers) + # track_obj, track_created = save_track_obj(track_dict['track'], + # track_artists, None) + # history_obj = save_history_obj(user_obj, parse(track_dict['played_at']), + # track_obj) -def parse_history(request, user_secret): - """Scans user's listening history and stores the information in a - database. + # if console_logging: + # tracks_processed += 1 + # print("Added history track #{}: {}".format( + # tracks_processed, history_obj,)) - :user_secret: secret for User object who's library is being scanned. - :returns: None - """ + # if len(artist_genre_queue) > 0: + # add_artist_genres(user_headers, artist_genre_queue) - user_obj = User.objects.get(secret=user_secret) - payload = {'limit': str(USER_TRACKS_LIMIT)} - last_time_played = History.objects.filter(user=user_obj).aggregate(Max('timestamp'))['timestamp__max'] - if last_time_played is not None: - payload['after'] = last_time_played.isoformat() - artist_genre_queue = [] - user_headers = get_user_header(user_obj) - history_response = requests.get(HISTORY_ENDPOINT, - headers=user_headers, - params=payload).json()['items'] - # pprint(history_response) + # # TODO: update track genres from History relation + # # update_track_genres(user_obj) - if console_logging: - tracks_processed = 0 + # return render(request, 'graphs/logged_in.html', get_user_context(user_obj)) - for track_dict in history_response: - # don't associate history track with User, not necessarily in their - # library - # track_obj, track_created = save_track_obj(track_dict['track'], - # track_artists, None) - track_artists = save_track_artists(track_dict['track'], artist_genre_queue, - user_headers) - track_obj, track_created = save_track_obj(track_dict['track'], - track_artists, None) - history_obj = save_history_obj(user_obj, parse(track_dict['played_at']), - track_obj) +# # }}} get_history # - if console_logging: - tracks_processed += 1 - print("Added history track #{}: {}".format( - tracks_processed, history_obj,)) - - if len(artist_genre_queue) > 0: - add_artist_genres(user_headers, artist_genre_queue) +# parse_history {{{ # - # TODO: update track genres from History relation - # update_track_genres(user_obj) +def parse_history_request(request, user_secret): + """Request function to call parse_history. Scans user's listening history + and stores the information in a database. - return render(request, 'graphs/logged_in.html', get_user_context(user_obj)) + :user_secret: secret for User object who's library is being scanned. + :returns: redirects user to logged in page + """ + parse_history(user_secret) + return render(request, 'graphs/logged_in.html', + get_user_context(User.objects.get(secret=user_secret))) # }}} get_history # From 4ddf11aa41cb0bde3f5409de62f9693199b65d99 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Sat, 17 Nov 2018 19:37:51 -0500 Subject: [PATCH 20/20] Added admin command to scan personal history (#58) Reverted exported history file to exclude timestamp. Simplified logging for scanning user history. --- .gitignore | 1 + api/management/commands/update-history.py | 10 +++++ api/utils.py | 13 +++--- api/views.py | 54 +---------------------- graphs/views.py | 5 ++- update-history.sh | 1 + 6 files changed, 24 insertions(+), 60 deletions(-) create mode 100644 api/management/commands/update-history.py create mode 100755 update-history.sh diff --git a/.gitignore b/.gitignore index 73bb973..d7ce0a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +*.log db.sqlite3 *.bak .idea/ diff --git a/api/management/commands/update-history.py b/api/management/commands/update-history.py new file mode 100644 index 0000000..0e48296 --- /dev/null +++ b/api/management/commands/update-history.py @@ -0,0 +1,10 @@ +from django.core.management.base import BaseCommand, CommandError +from api.utils import parse_history +from login.models import User + +class Command(BaseCommand): + help = 'Update history for users who requested it' + + def handle(self, *args, **options): + user_id = "polarbier" + parse_history(User.objects.get(id=user_id).secret) diff --git a/api/utils.py b/api/utils.py index 9f24864..6ac7995 100644 --- a/api/utils.py +++ b/api/utils.py @@ -13,13 +13,14 @@ from . import views from login.models import User from pprint import pprint from dateutil.parser import parse +from datetime import datetime HISTORY_ENDPOINT = 'https://api.spotify.com/v1/me/player/recently-played' # }}} imports # -console_logging = True -# console_logging = False +# console_logging = True +console_logging = False artists_genre_processed = 0 features_processed = 0 @@ -365,8 +366,7 @@ def parse_history(user_secret): params=payload).json()['items'] # pprint(history_response) - if console_logging: - tracks_processed = 0 + tracks_processed = 0 for track_dict in history_response: # don't associate history track with User, not necessarily in their @@ -379,9 +379,9 @@ def parse_history(user_secret): track_artists, None) history_obj = save_history_obj(user_obj, parse(track_dict['played_at']), track_obj) + tracks_processed += 1 if console_logging: - tracks_processed += 1 print("Added history track #{}: {}".format( tracks_processed, history_obj,)) @@ -391,5 +391,8 @@ def parse_history(user_secret): # TODO: update track genres from History relation # update_track_genres(user_obj) + print("Scanned {} history tracks for user {} at {}.".format( + tracks_processed, user_obj.id, datetime.now())) + # }}} get_history # diff --git a/api/views.py b/api/views.py index bd6df30..17e20b4 100644 --- a/api/views.py +++ b/api/views.py @@ -115,59 +115,7 @@ def parse_library(request, user_secret): # }}} parse_library # -# # parse_history {{{ # - -# def parse_history(request, user_secret): - # """Scans user's listening history and stores the information in a - # database. - - # :user_secret: secret for User object who's library is being scanned. - # :returns: None - # """ - - # user_obj = User.objects.get(secret=user_secret) - # payload = {'limit': str(USER_TRACKS_LIMIT)} - # last_time_played = History.objects.filter(user=user_obj).aggregate(Max('timestamp'))['timestamp__max'] - # if last_time_played is not None: - # payload['after'] = last_time_played.isoformat() - # artist_genre_queue = [] - # user_headers = get_user_header(user_obj) - # history_response = requests.get(HISTORY_ENDPOINT, - # headers=user_headers, - # params=payload).json()['items'] - # # pprint(history_response) - - # if console_logging: - # tracks_processed = 0 - - # for track_dict in history_response: - # # don't associate history track with User, not necessarily in their - # # library - # # track_obj, track_created = save_track_obj(track_dict['track'], - # # track_artists, None) - # track_artists = save_track_artists(track_dict['track'], artist_genre_queue, - # user_headers) - # track_obj, track_created = save_track_obj(track_dict['track'], - # track_artists, None) - # history_obj = save_history_obj(user_obj, parse(track_dict['played_at']), - # track_obj) - - # if console_logging: - # tracks_processed += 1 - # print("Added history track #{}: {}".format( - # tracks_processed, history_obj,)) - - # if len(artist_genre_queue) > 0: - # add_artist_genres(user_headers, artist_genre_queue) - - # # TODO: update track genres from History relation - # # update_track_genres(user_obj) - - # return render(request, 'graphs/logged_in.html', get_user_context(user_obj)) - -# # }}} get_history # - -# parse_history {{{ # +# parse_history_request {{{ # def parse_history_request(request, user_secret): """Request function to call parse_history. Scans user's listening history diff --git a/graphs/views.py b/graphs/views.py index 8c410d6..4b5278a 100644 --- a/graphs/views.py +++ b/graphs/views.py @@ -71,8 +71,9 @@ class HistoryList(ExportMixin, SingleTableView): def get_export_filename(self, export_format): user_id = self.request.session['user_id'] - timestamp = strftime("%m%d%Y-%H%M") - return "{}.{}".format("-".join((user_id, timestamp)), export_format) + # timestamp = strftime("%m%d%Y-%H%M") + # return "{}.{}".format("-".join((user_id, timestamp)), export_format) + return "{}.{}".format(user_id, export_format) def create_export(self, export_format): export_exclude = ('id', 'user', 'track', 'track_name', 'artists', diff --git a/update-history.sh b/update-history.sh new file mode 100755 index 0000000..5f2c10d --- /dev/null +++ b/update-history.sh @@ -0,0 +1 @@ +/home/kevin/coding/spotify-lib-vis/bin/python /home/kevin/coding/spotify-lib-vis/src/manage.py update-history >> /home/kevin/coding/spotify-lib-vis/src/api/management/commands/update-history.log