From c949ecd3cc07355a2b3a2a6b575f57560d0ad020 Mon Sep 17 00:00:00 2001 From: Kevin Mok Date: Fri, 29 Jun 2018 10:15:29 -0400 Subject: [PATCH] Setup API app, can scan library (#47) Manually merged console logging from loading-page branch. --- api/models.py | 7 +-- api/templates/api/logged_in.html | 8 +-- api/urls.py | 3 +- api/utils.py | 93 ++++++++++++++++++++------- api/views.py | 80 ++++++++++++++---------- graphs/models.py | 104 ------------------------------- login/models.py | 4 +- login/templates/login/scan.html | 7 ++- login/views.py | 35 ----------- reset_db.sh | 1 + 10 files changed, 135 insertions(+), 207 deletions(-) delete mode 100644 graphs/models.py diff --git a/api/models.py b/api/models.py index 5303876..3a4d321 100644 --- a/api/models.py +++ b/api/models.py @@ -22,15 +22,14 @@ class Genre(models.Model): # Artist {{{ # - class Artist(models.Model): class Meta: verbose_name = "Artist" verbose_name_plural = "Artists" - artist_id = models.CharField(primary_key=True, max_length=MAX_ID) + id = models.CharField(primary_key=True, max_length=MAX_ID) # unique since only storing one genre per artist right now - name = models.CharField(unique=True, max_length=50) + name = models.CharField(max_length=50) genres = models.ManyToManyField(Genre, blank=True) def __str__(self): @@ -46,7 +45,7 @@ class Track(models.Model): verbose_name = "Track" verbose_name_plural = "Tracks" - track_id = models.CharField(primary_key=True, max_length=MAX_ID) + id = models.CharField(primary_key=True, max_length=MAX_ID) # artist = models.ForeignKey(Artist, on_delete=models.CASCADE) artists = models.ManyToManyField(Artist, blank=True) year = models.PositiveSmallIntegerField() diff --git a/api/templates/api/logged_in.html b/api/templates/api/logged_in.html index 1df0826..b6e4a69 100644 --- a/api/templates/api/logged_in.html +++ b/api/templates/api/logged_in.html @@ -5,15 +5,15 @@ Logged In - +

{{ user_id }}'s Graphs

- Audio Features - Genres - + Artists diff --git a/api/urls.py b/api/urls.py index 1196097..60126f6 100644 --- a/api/urls.py +++ b/api/urls.py @@ -4,7 +4,8 @@ from .views import * app_name = 'api' urlpatterns = [ - # path('scan/', get_artist_data), + path('scan/', parse_library, + name='scan'), path('user_artists/', get_artist_data, name='get_artist_data'), path('user_genres/', get_genre_data, diff --git a/api/utils.py b/api/utils.py index a3d1b22..a090a1b 100644 --- a/api/utils.py +++ b/api/utils.py @@ -2,33 +2,36 @@ import requests import math import pprint +import os +import json -from .models import * from django.db.models import Count, Q, F from django.http import JsonResponse from django.core import serializers -import json +from django.utils import timezone +from .models import * +from login.models import User # }}} imports # -USER_TRACKS_LIMIT = 50 -ARTIST_LIMIT = 50 -FEATURES_LIMIT = 100 -# ARTIST_LIMIT = 25 -# FEATURES_LIMIT = 25 +console_logging = True +# console_logging = False +artists_genre_processed = 0 +features_processed = 0 # update_track_genres {{{ # -def update_track_genres(user): - """Updates user's tracks with the most common genre associated with the +def update_track_genres(user_obj): + """Updates user_obj's tracks with the most common genre associated with the songs' artist(s). - :user: User object who's tracks are being updated. + :user_obj: User object who's tracks are being updated. :returns: None """ - user_tracks = Track.objects.filter(users__exact=user) + tracks_processed = 0 + user_tracks = Track.objects.filter(users__exact=user_obj) for track in user_tracks: # just using this variable to save another call to db track_artists = track.artists.all() @@ -44,41 +47,46 @@ def update_track_genres(user): track.genre = most_common_genre if most_common_genre is not None \ else undefined_genre_obj track.save() - # print(track.name, track.genre) + tracks_processed += 1 + + if console_logging: + print("Added '{}' as genre for song #{} - '{}'".format( + track.genre, + tracks_processed, + track.name, + )) # }}} update_track_genres # # save_track_obj {{{ # -def save_track_obj(track_dict, artists, top_genre, user): +def save_track_obj(track_dict, artists, user_obj): """Make an entry in the database for this track if it doesn't exist already. :track_dict: dictionary from the API call containing track information. :artists: artists of the song, passed in as a list of Artist objects. - :top_genre: top genre associated with this track (see get_top_genre). - :user: User object for which this Track is to be associated with. + :user_obj: User object for which this Track is to be associated with. :returns: (The created/retrieved Track object, created) """ - track_query = Track.objects.filter(track_id__exact=track_dict['id']) + track_query = Track.objects.filter(id__exact=track_dict['id']) if len(track_query) != 0: return track_query[0], False else: new_track = Track.objects.create( - track_id=track_dict['id'], + id=track_dict['id'], year=track_dict['album']['release_date'].split('-')[0], popularity=int(track_dict['popularity']), runtime=int(float(track_dict['duration_ms']) / 1000), name=track_dict['name'], - # genre=top_genre, ) - # have to add artists and user after saving object since track needs to + # have to add artists and user_obj after saving object since track needs to # have ID before filling in m2m field for artist in artists: new_track.artists.add(artist) - new_track.users.add(user) + new_track.users.add(user_obj) new_track.save() return new_track, True @@ -96,13 +104,14 @@ def get_audio_features(headers, track_objs): :returns: None """ - track_ids = str.join(",", [track_obj.track_id for track_obj in track_objs]) + track_ids = str.join(",", [track_obj.id for track_obj in track_objs]) params = {'ids': track_ids} features_response = requests.get("https://api.spotify.com/v1/audio-features", headers=headers,params=params).json()['audio_features'] # pprint.pprint(features_response) - useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", ] + useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", + "track_href", "analysis_url", "time_signature", ] for i in range(len(track_objs)): if features_response[i] is not None: # Data that we don't need @@ -113,6 +122,12 @@ def get_audio_features(headers, track_objs): setattr(cur_features_obj, key, val) cur_features_obj.save() + if console_logging: + global features_processed + features_processed += 1 + print("Added features for song #{} - {}".format( + features_processed, track_objs[i].name)) + # }}} get_audio_features # def process_artist_genre(genre_name, artist_obj): @@ -145,7 +160,7 @@ def add_artist_genres(headers, artist_objs): :returns: None """ - artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs]) + artist_ids = str.join(",", [artist_obj.id for artist_obj in artist_objs]) params = {'ids': artist_ids} artists_response = requests.get('https://api.spotify.com/v1/artists/', headers=headers, params=params).json()['artists'] @@ -157,6 +172,12 @@ def add_artist_genres(headers, artist_objs): for genre in artists_response[i]['genres']: process_artist_genre(genre, artist_objs[i]) + if console_logging: + global artists_genre_processed + artists_genre_processed += 1 + print("Added genres for artist #{} - {}".format( + artists_genre_processed, artist_objs[i].name)) + # }}} add_artist_genres # # get_artists_in_genre {{{ # @@ -192,3 +213,29 @@ def get_artists_in_genre(user, genre, max_songs): return processed_artist_counts # }}} get_artists_in_genre # + +def get_user_header(user_obj): + """Returns the authorization string needed to make an API call. + + :user_obj: User to return the auth string for. + :returns: the authorization string used for the header in a Spotify API + call. + + """ + seconds_elapsed = (timezone.now() - + user_obj.access_obtained_at).total_seconds() + if seconds_elapsed >= user_obj.access_expires_in: + req_body = { + 'grant_type': 'refresh_token', + 'refresh_token': user_obj.refresh_token, + 'client_id': os.environ['SPOTIFY_CLIENT_ID'], + 'client_secret': os.environ['SPOTIFY_CLIENT_SECRET'] + } + + token_response = requests.post('https://accounts.spotify.com/api/token', + data=req_body).json() + user_obj.access_token = token_response['access_token'] + user_obj.access_expires_in = token_response['expires_in'] + user_obj.save() + + return {'Authorization': "Bearer " + user_obj.access_token} diff --git a/api/views.py b/api/views.py index 92539d4..0fb13df 100644 --- a/api/views.py +++ b/api/views.py @@ -3,49 +3,59 @@ import math import random import requests -import os import urllib import secrets import pprint import string -from datetime import datetime +from django.shortcuts import render, redirect from django.http import JsonResponse from django.db.models import Count, Q -from .utils import get_artists_in_genre, update_track_genres -from .models import User, Track, AudioFeatures, Artist +from .utils import * +from .models import * +from login.models import User # }}} imports # +USER_TRACKS_LIMIT = 50 +ARTIST_LIMIT = 50 +FEATURES_LIMIT = 100 +# ARTIST_LIMIT = 25 +# FEATURES_LIMIT = 25 TRACKS_TO_QUERY = 200 -# parse_library {{{ # +console_logging = True -def parse_library(headers, tracks, user): - """Scans user's library for certain number of tracks and store the information in a database +# parse_library {{{ # - :headers: For API call. - :tracks: Number of tracks to get from user's library. - :user: a User object representing the user whose library we are parsing +def parse_library(request, user_secret): + """Scans user's library for num_tracks and store the information in a + database. + :user_secret: secret for User object who's library is being scanned. :returns: None - """ - # TODO: implement importing entire library with 0 as tracks param - # keeps track of point to get songs from + offset = 0 payload = {'limit': str(USER_TRACKS_LIMIT)} artist_genre_queue = [] features_queue = [] + user_obj = User.objects.get(secret=user_secret) + user_headers = get_user_header(user_obj) - # iterate until hit requested num of tracks - for i in range(0, tracks, USER_TRACKS_LIMIT): + # create this obj so loop runs at least once + saved_tracks_response = [0] + # scan until reach num_tracks or no tracks left if scanning entire library + while (TRACKS_TO_QUERY == 0 or offset < TRACKS_TO_QUERY) and len(saved_tracks_response) > 0: payload['offset'] = str(offset) saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', - headers=headers, - params=payload).json() + headers=user_headers, + params=payload).json()['items'] + + if console_logging: + tracks_processed = 0 - for track_dict in saved_tracks_response['items']: + for track_dict in saved_tracks_response: # add artists {{{ # # update artist info before track so that Track object can reference @@ -53,22 +63,20 @@ def parse_library(headers, tracks, user): track_artists = [] for artist_dict in track_dict['track']['artists']: artist_obj, artist_created = Artist.objects.get_or_create( - artist_id=artist_dict['id'], + id=artist_dict['id'], name=artist_dict['name'],) # only add/tally up artist genres if new if artist_created: artist_genre_queue.append(artist_obj) if len(artist_genre_queue) == ARTIST_LIMIT: - add_artist_genres(headers, artist_genre_queue) + add_artist_genres(user_headers, artist_genre_queue) artist_genre_queue = [] track_artists.append(artist_obj) # }}} add artists # - # TODO: fix this, don't need any more - top_genre = "" track_obj, track_created = save_track_obj(track_dict['track'], - track_artists, top_genre, user) + track_artists, user_obj) # add audio features {{{ # @@ -77,16 +85,18 @@ def parse_library(headers, tracks, user): if track_created: features_queue.append(track_obj) if len(features_queue) == FEATURES_LIMIT: - get_audio_features(headers, features_queue) + get_audio_features(user_headers, features_queue) features_queue = [] # }}} add audio features # - # temporary console logging - print("#{}-{}: {} - {}".format(offset + 1, - offset + USER_TRACKS_LIMIT, - track_obj.artists.first(), - track_obj.name)) + if console_logging: + tracks_processed += 1 + print("Added track #{}: {} - {}".format( + offset + tracks_processed, + track_obj.artists.first(), + track_obj.name, + )) # calculates num_songs with offset + songs retrieved offset += USER_TRACKS_LIMIT @@ -96,13 +106,19 @@ def parse_library(headers, tracks, user): # update remaining artists without genres and songs without features if # there are any if len(artist_genre_queue) > 0: - add_artist_genres(headers, artist_genre_queue) + add_artist_genres(user_headers, artist_genre_queue) if len(features_queue) > 0: - get_audio_features(headers, features_queue) + get_audio_features(user_headers, features_queue) # }}} clean-up # - update_track_genres(user) + update_track_genres(user_obj) + + context = { + 'user_id': user_obj.id, + 'user_secret': user_obj.secret, + } + return render(request, 'api/logged_in.html', context) # }}} parse_library # diff --git a/graphs/models.py b/graphs/models.py deleted file mode 100644 index 7419443..0000000 --- a/graphs/models.py +++ /dev/null @@ -1,104 +0,0 @@ -from django.db import models - -# id's are 22 in length in examples but set to 30 for buffer -MAX_ID = 30 - -# Genre {{{ # - -class Genre(models.Model): - - class Meta: - verbose_name = "Genre" - verbose_name_plural = "Genres" - - name = models.CharField(primary_key=True, max_length=50) - num_songs = models.PositiveIntegerField() - - def __str__(self): - return self.name - -# }}} Genre # - -# Artist {{{ # - - -class Artist(models.Model): - class Meta: - verbose_name = "Artist" - verbose_name_plural = "Artists" - - artist_id = models.CharField(primary_key=True, max_length=MAX_ID) - # unique since only storing one genre per artist right now - name = models.CharField(unique=True, max_length=50) - genres = models.ManyToManyField(Genre, blank=True) - - def __str__(self): - return self.name - -# }}} Artist # - -# User {{{ # - -class User(models.Model): - class Meta: - verbose_name = "User" - verbose_name_plural = "Users" - - user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID - user_secret = models.CharField(max_length=50, default='') - - def __str__(self): - return self.user_id - -# }}} User # - -# Track {{{ # - -class Track(models.Model): - - class Meta: - verbose_name = "Track" - verbose_name_plural = "Tracks" - - track_id = models.CharField(primary_key=True, max_length=MAX_ID) - # artist = models.ForeignKey(Artist, on_delete=models.CASCADE) - artists = models.ManyToManyField(Artist, blank=True) - year = models.PositiveSmallIntegerField() - popularity = models.PositiveSmallIntegerField() - runtime = models.PositiveSmallIntegerField() - name = models.CharField(max_length=200) - users = models.ManyToManyField(User, blank=True) - genre = models.ForeignKey(Genre, on_delete=models.CASCADE, blank=True, - null=True) - - def __str__(self): - track_str = "{}, genre: {}, artists: [".format(self.name, self.genre) - for artist in self.artists.all(): - track_str += "{}, ".format(artist.name) - track_str += "]" - return track_str - -# }}} Track # - -# AudioFeatures {{{ # - -class AudioFeatures(models.Model): - - class Meta: - verbose_name = "AudioFeatures" - verbose_name_plural = "AudioFeatures" - - track = models.OneToOneField(Track, on_delete=models.CASCADE, primary_key=True,) - acousticness = models.DecimalField(decimal_places=3, max_digits=3) - danceability = models.DecimalField(decimal_places=3, max_digits=3) - energy = models.DecimalField(decimal_places=3, max_digits=3) - instrumentalness = models.DecimalField(decimal_places=3, max_digits=3) - loudness = models.DecimalField(decimal_places=3, max_digits=6) - speechiness = models.DecimalField(decimal_places=3, max_digits=3) - tempo = models.DecimalField(decimal_places=3, max_digits=6) - valence = models.DecimalField(decimal_places=3, max_digits=3) - - def __str__(self): - return super(AudioFeatures, self).__str__() - -# }}} AudioFeatures # diff --git a/login/models.py b/login/models.py index e3fa787..aa04baf 100644 --- a/login/models.py +++ b/login/models.py @@ -15,8 +15,8 @@ class User(models.Model): secret = models.CharField(max_length=50, default='') refresh_token = models.CharField(max_length=TOKEN_LENGTH) access_token = models.CharField(max_length=TOKEN_LENGTH) - access_obtained_at = models.DateTimeField(auto_now_add=True) + access_obtained_at = models.DateTimeField(auto_now=True) access_expires_in = models.PositiveIntegerField() def __str__(self): - return self.user_id + return self.id diff --git a/login/templates/login/scan.html b/login/templates/login/scan.html index 6bc8244..183742c 100644 --- a/login/templates/login/scan.html +++ b/login/templates/login/scan.html @@ -10,13 +10,16 @@ User Spotify Data + -

Logged in as {{ user_id }}

- Scan Library +

Logged in as {{ user_id }}

+ + Scan Library + diff --git a/login/views.py b/login/views.py index 9af5079..74e0188 100644 --- a/login/views.py +++ b/login/views.py @@ -38,20 +38,6 @@ def generate_random_string(length): # }}} generate_random_string # -# token_expired {{{ # - -def token_expired(token_obtained_at, valid_for): - """Returns True if token expired, False if otherwise - - Args: - token_obtained_at: datetime object representing the date and time when the token was obtained - valid_for: the time duration for which the token is valid, in seconds - """ - time_elapsed = (datetime.today() - token_obtained_at).total_seconds() - return time_elapsed >= valid_for - -# }}} token_expired # - # index {{{ # # Create your views here. @@ -146,27 +132,6 @@ def create_user(refresh_token, access_token, access_expires_in): return user_obj -# refresh access token {{{ # - -""" -token_obtained_at = datetime.strptime(request.session['token_obtained_at'], TIME_FORMAT) -valid_for = int(request.session['valid_for']) - -if token_expired(token_obtained_at, valid_for): - req_body = { - 'grant_type': 'refresh_token', - 'refresh_token': request.session['refresh_token'], - 'client_id': os.environ['SPOTIFY_CLIENT_ID'], - 'client_secret': os.environ['SPOTIFY_CLIENT_SECRET'] - } - - refresh_token_response = requests.post('https://accounts.spotify.com/api/token', data=req_body).json() - request.session['access_token'] = refresh_token_response['access_token'] - request.session['valid_for'] = refresh_token_response['expires_in'] -""" - -# }}} refresh access token # - # admin_graphs {{{ # def admin_graphs(request): diff --git a/reset_db.sh b/reset_db.sh index 7e3e7c7..51037df 100755 --- a/reset_db.sh +++ b/reset_db.sh @@ -11,4 +11,5 @@ rm login/migrations/0* api/migrations/0* graphs/migrations/0* sudo -u postgres psql -f reset_db.sql python manage.py makemigrations python manage.py migrate +python manage.py runserver # fi