Kevin-Mok
/
spotify-lib-vis
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

Setup API app, can scan library (#47) 

Manually merged console logging from loading-page branch.
master
Kevin Mok 6 years ago
parent
e50d3d8476
commit
c949ecd3cc
10 changed files with 135 additions and 207 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 7
      api/models.py
  2. 8
      api/templates/api/logged_in.html
  3. 3
      api/urls.py
  4. 93
      api/utils.py
  5. 80
      api/views.py
  6. 104
      graphs/models.py
  7. 4
      login/models.py
  8. 7
      login/templates/login/scan.html
  9. 35
      login/views.py
  10. 1
      reset_db.sh

7
api/models.py
View File

@ -22,15 +22,14 @@ class Genre(models.Model):
# Artist {{{ # # Artist {{{ #
class Artist(models.Model): class Artist(models.Model):
class Meta: class Meta:
verbose_name = "Artist" verbose_name = "Artist"
verbose_name_plural = "Artists" verbose_name_plural = "Artists"
artist_id = models.CharField(primary_key=True, max_length=MAX_ID)
id = models.CharField(primary_key=True, max_length=MAX_ID)
# unique since only storing one genre per artist right now # unique since only storing one genre per artist right now
name = models.CharField(unique=True, max_length=50)
name = models.CharField(max_length=50)
genres = models.ManyToManyField(Genre, blank=True) genres = models.ManyToManyField(Genre, blank=True)
def __str__(self): def __str__(self):
@ -46,7 +45,7 @@ class Track(models.Model):
verbose_name = "Track" verbose_name = "Track"
verbose_name_plural = "Tracks" verbose_name_plural = "Tracks"
track_id = models.CharField(primary_key=True, max_length=MAX_ID)
id = models.CharField(primary_key=True, max_length=MAX_ID)
# artist = models.ForeignKey(Artist, on_delete=models.CASCADE) # artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
artists = models.ManyToManyField(Artist, blank=True) artists = models.ManyToManyField(Artist, blank=True)
year = models.PositiveSmallIntegerField() year = models.PositiveSmallIntegerField()

8
api/templates/api/logged_in.html
View File

@ -5,15 +5,15 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<title>Logged In</title> <title>Logged In</title>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous"> <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
<link rel="stylesheet" href="{% static 'spotifyvis/css/dark_bg.css' %}">
<link rel="stylesheet" href="{% static 'css/dark_bg.css' %}">
</head> </head>
<body> <body>
<h1>{{ user_id }}'s Graphs</h1> <h1>{{ user_id }}'s Graphs</h1>
<a class="btn btn-primary" href="{% url "display_audio_features" user_secret %}"
<a class="btn btn-primary" href=""
role="button">Audio Features</a> role="button">Audio Features</a>
<a class="btn btn-primary" href="{% url "display_genre_graph" user_secret %}"
<a class="btn btn-primary" href=""
role="button">Genres</a> role="button">Genres</a>
<a class="btn btn-primary" href="{% url "display_artist_graph" user_secret %}" role="button">
<a class="btn btn-primary" href="" role="button">
Artists Artists
</a> </a>
</body> </body>

3
api/urls.py
View File

@ -4,7 +4,8 @@ from .views import *
app_name = 'api' app_name = 'api'
urlpatterns = [ urlpatterns = [
# path('scan/<str:user_secret>', get_artist_data),
path('scan/<str:user_secret>', parse_library,
name='scan'),
path('user_artists/<str:user_secret>', get_artist_data, path('user_artists/<str:user_secret>', get_artist_data,
name='get_artist_data'), name='get_artist_data'),
path('user_genres/<str:user_secret>', get_genre_data, path('user_genres/<str:user_secret>', get_genre_data,

93
api/utils.py
View File

@ -2,33 +2,36 @@
import requests import requests
import math import math
import pprint import pprint
import os
import json
from .models import *
from django.db.models import Count, Q, F from django.db.models import Count, Q, F
from django.http import JsonResponse from django.http import JsonResponse
from django.core import serializers from django.core import serializers
import json
from django.utils import timezone
from .models import *
from login.models import User
# }}} imports # # }}} imports #
USER_TRACKS_LIMIT = 50
ARTIST_LIMIT = 50
FEATURES_LIMIT = 100
# ARTIST_LIMIT = 25
# FEATURES_LIMIT = 25
console_logging = True
# console_logging = False
artists_genre_processed = 0
features_processed = 0
# update_track_genres {{{ # # update_track_genres {{{ #
def update_track_genres(user):
"""Updates user's tracks with the most common genre associated with the
def update_track_genres(user_obj):
"""Updates user_obj's tracks with the most common genre associated with the
songs' artist(s). songs' artist(s).
:user: User object who's tracks are being updated.
:user_obj: User object who's tracks are being updated.
:returns: None :returns: None
""" """
user_tracks = Track.objects.filter(users__exact=user)
tracks_processed = 0
user_tracks = Track.objects.filter(users__exact=user_obj)
for track in user_tracks: for track in user_tracks:
# just using this variable to save another call to db # just using this variable to save another call to db
track_artists = track.artists.all() track_artists = track.artists.all()
@ -44,41 +47,46 @@ def update_track_genres(user):
track.genre = most_common_genre if most_common_genre is not None \ track.genre = most_common_genre if most_common_genre is not None \
else undefined_genre_obj else undefined_genre_obj
track.save() track.save()
# print(track.name, track.genre)
tracks_processed += 1
if console_logging:
print("Added '{}' as genre for song #{} - '{}'".format(
track.genre,
tracks_processed,
track.name,
))
# }}} update_track_genres # # }}} update_track_genres #
# save_track_obj {{{ # # save_track_obj {{{ #
def save_track_obj(track_dict, artists, top_genre, user):
def save_track_obj(track_dict, artists, user_obj):
"""Make an entry in the database for this track if it doesn't exist already. """Make an entry in the database for this track if it doesn't exist already.
:track_dict: dictionary from the API call containing track information. :track_dict: dictionary from the API call containing track information.
:artists: artists of the song, passed in as a list of Artist objects. :artists: artists of the song, passed in as a list of Artist objects.
:top_genre: top genre associated with this track (see get_top_genre).
:user: User object for which this Track is to be associated with.
:user_obj: User object for which this Track is to be associated with.
:returns: (The created/retrieved Track object, created) :returns: (The created/retrieved Track object, created)
""" """
track_query = Track.objects.filter(track_id__exact=track_dict['id'])
track_query = Track.objects.filter(id__exact=track_dict['id'])
if len(track_query) != 0: if len(track_query) != 0:
return track_query[0], False return track_query[0], False
else: else:
new_track = Track.objects.create( new_track = Track.objects.create(
track_id=track_dict['id'],
id=track_dict['id'],
year=track_dict['album']['release_date'].split('-')[0], year=track_dict['album']['release_date'].split('-')[0],
popularity=int(track_dict['popularity']), popularity=int(track_dict['popularity']),
runtime=int(float(track_dict['duration_ms']) / 1000), runtime=int(float(track_dict['duration_ms']) / 1000),
name=track_dict['name'], name=track_dict['name'],
# genre=top_genre,
) )
# have to add artists and user after saving object since track needs to
# have to add artists and user_obj after saving object since track needs to
# have ID before filling in m2m field # have ID before filling in m2m field
for artist in artists: for artist in artists:
new_track.artists.add(artist) new_track.artists.add(artist)
new_track.users.add(user)
new_track.users.add(user_obj)
new_track.save() new_track.save()
return new_track, True return new_track, True
@ -96,13 +104,14 @@ def get_audio_features(headers, track_objs):
:returns: None :returns: None
""" """
track_ids = str.join(",", [track_obj.track_id for track_obj in track_objs])
track_ids = str.join(",", [track_obj.id for track_obj in track_objs])
params = {'ids': track_ids} params = {'ids': track_ids}
features_response = requests.get("https://api.spotify.com/v1/audio-features", features_response = requests.get("https://api.spotify.com/v1/audio-features",
headers=headers,params=params).json()['audio_features'] headers=headers,params=params).json()['audio_features']
# pprint.pprint(features_response) # pprint.pprint(features_response)
useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", ]
useless_keys = [ "key", "mode", "type", "liveness", "id", "uri",
"track_href", "analysis_url", "time_signature", ]
for i in range(len(track_objs)): for i in range(len(track_objs)):
if features_response[i] is not None: if features_response[i] is not None:
# Data that we don't need # Data that we don't need
@ -113,6 +122,12 @@ def get_audio_features(headers, track_objs):
setattr(cur_features_obj, key, val) setattr(cur_features_obj, key, val)
cur_features_obj.save() cur_features_obj.save()
if console_logging:
global features_processed
features_processed += 1
print("Added features for song #{} - {}".format(
features_processed, track_objs[i].name))
# }}} get_audio_features # # }}} get_audio_features #
def process_artist_genre(genre_name, artist_obj): def process_artist_genre(genre_name, artist_obj):
@ -145,7 +160,7 @@ def add_artist_genres(headers, artist_objs):
:returns: None :returns: None
""" """
artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs])
artist_ids = str.join(",", [artist_obj.id for artist_obj in artist_objs])
params = {'ids': artist_ids} params = {'ids': artist_ids}
artists_response = requests.get('https://api.spotify.com/v1/artists/', artists_response = requests.get('https://api.spotify.com/v1/artists/',
headers=headers, params=params).json()['artists'] headers=headers, params=params).json()['artists']
@ -157,6 +172,12 @@ def add_artist_genres(headers, artist_objs):
for genre in artists_response[i]['genres']: for genre in artists_response[i]['genres']:
process_artist_genre(genre, artist_objs[i]) process_artist_genre(genre, artist_objs[i])
if console_logging:
global artists_genre_processed
artists_genre_processed += 1
print("Added genres for artist #{} - {}".format(
artists_genre_processed, artist_objs[i].name))
# }}} add_artist_genres # # }}} add_artist_genres #
# get_artists_in_genre {{{ # # get_artists_in_genre {{{ #
@ -192,3 +213,29 @@ def get_artists_in_genre(user, genre, max_songs):
return processed_artist_counts return processed_artist_counts
# }}} get_artists_in_genre # # }}} get_artists_in_genre #
def get_user_header(user_obj):
"""Returns the authorization string needed to make an API call.
:user_obj: User to return the auth string for.
:returns: the authorization string used for the header in a Spotify API
call.
"""
seconds_elapsed = (timezone.now() -
user_obj.access_obtained_at).total_seconds()
if seconds_elapsed >= user_obj.access_expires_in:
req_body = {
'grant_type': 'refresh_token',
'refresh_token': user_obj.refresh_token,
'client_id': os.environ['SPOTIFY_CLIENT_ID'],
'client_secret': os.environ['SPOTIFY_CLIENT_SECRET']
}
token_response = requests.post('https://accounts.spotify.com/api/token',
data=req_body).json()
user_obj.access_token = token_response['access_token']
user_obj.access_expires_in = token_response['expires_in']
user_obj.save()
return {'Authorization': "Bearer " + user_obj.access_token}

80
api/views.py
View File

@ -3,49 +3,59 @@
import math import math
import random import random
import requests import requests
import os
import urllib import urllib
import secrets import secrets
import pprint import pprint
import string import string
from datetime import datetime
from django.shortcuts import render, redirect
from django.http import JsonResponse from django.http import JsonResponse
from django.db.models import Count, Q from django.db.models import Count, Q
from .utils import get_artists_in_genre, update_track_genres
from .models import User, Track, AudioFeatures, Artist
from .utils import *
from .models import *
from login.models import User
# }}} imports # # }}} imports #
USER_TRACKS_LIMIT = 50
ARTIST_LIMIT = 50
FEATURES_LIMIT = 100
# ARTIST_LIMIT = 25
# FEATURES_LIMIT = 25
TRACKS_TO_QUERY = 200 TRACKS_TO_QUERY = 200
# parse_library {{{ #
console_logging = True
def parse_library(headers, tracks, user):
"""Scans user's library for certain number of tracks and store the information in a database
# parse_library {{{ #
:headers: For API call.
:tracks: Number of tracks to get from user's library.
:user: a User object representing the user whose library we are parsing
def parse_library(request, user_secret):
"""Scans user's library for num_tracks and store the information in a
database.
:user_secret: secret for User object who's library is being scanned.
:returns: None :returns: None
""" """
# TODO: implement importing entire library with 0 as tracks param
# keeps track of point to get songs from
offset = 0 offset = 0
payload = {'limit': str(USER_TRACKS_LIMIT)} payload = {'limit': str(USER_TRACKS_LIMIT)}
artist_genre_queue = [] artist_genre_queue = []
features_queue = [] features_queue = []
user_obj = User.objects.get(secret=user_secret)
user_headers = get_user_header(user_obj)
# iterate until hit requested num of tracks
for i in range(0, tracks, USER_TRACKS_LIMIT):
# create this obj so loop runs at least once
saved_tracks_response = [0]
# scan until reach num_tracks or no tracks left if scanning entire library
while (TRACKS_TO_QUERY == 0 or offset < TRACKS_TO_QUERY) and len(saved_tracks_response) > 0:
payload['offset'] = str(offset) payload['offset'] = str(offset)
saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks',
headers=headers,
params=payload).json()
headers=user_headers,
params=payload).json()['items']
if console_logging:
tracks_processed = 0
for track_dict in saved_tracks_response['items']:
for track_dict in saved_tracks_response:
# add artists {{{ # # add artists {{{ #
# update artist info before track so that Track object can reference # update artist info before track so that Track object can reference
@ -53,22 +63,20 @@ def parse_library(headers, tracks, user):
track_artists = [] track_artists = []
for artist_dict in track_dict['track']['artists']: for artist_dict in track_dict['track']['artists']:
artist_obj, artist_created = Artist.objects.get_or_create( artist_obj, artist_created = Artist.objects.get_or_create(
artist_id=artist_dict['id'],
id=artist_dict['id'],
name=artist_dict['name'],) name=artist_dict['name'],)
# only add/tally up artist genres if new # only add/tally up artist genres if new
if artist_created: if artist_created:
artist_genre_queue.append(artist_obj) artist_genre_queue.append(artist_obj)
if len(artist_genre_queue) == ARTIST_LIMIT: if len(artist_genre_queue) == ARTIST_LIMIT:
add_artist_genres(headers, artist_genre_queue)
add_artist_genres(user_headers, artist_genre_queue)
artist_genre_queue = [] artist_genre_queue = []
track_artists.append(artist_obj) track_artists.append(artist_obj)
# }}} add artists # # }}} add artists #
# TODO: fix this, don't need any more
top_genre = ""
track_obj, track_created = save_track_obj(track_dict['track'], track_obj, track_created = save_track_obj(track_dict['track'],
track_artists, top_genre, user)
track_artists, user_obj)
# add audio features {{{ # # add audio features {{{ #
@ -77,16 +85,18 @@ def parse_library(headers, tracks, user):
if track_created: if track_created:
features_queue.append(track_obj) features_queue.append(track_obj)
if len(features_queue) == FEATURES_LIMIT: if len(features_queue) == FEATURES_LIMIT:
get_audio_features(headers, features_queue)
get_audio_features(user_headers, features_queue)
features_queue = [] features_queue = []
# }}} add audio features # # }}} add audio features #
# temporary console logging
print("#{}-{}: {} - {}".format(offset + 1,
offset + USER_TRACKS_LIMIT,
track_obj.artists.first(),
track_obj.name))
if console_logging:
tracks_processed += 1
print("Added track #{}: {} - {}".format(
offset + tracks_processed,
track_obj.artists.first(),
track_obj.name,
))
# calculates num_songs with offset + songs retrieved # calculates num_songs with offset + songs retrieved
offset += USER_TRACKS_LIMIT offset += USER_TRACKS_LIMIT
@ -96,13 +106,19 @@ def parse_library(headers, tracks, user):
# update remaining artists without genres and songs without features if # update remaining artists without genres and songs without features if
# there are any # there are any
if len(artist_genre_queue) > 0: if len(artist_genre_queue) > 0:
add_artist_genres(headers, artist_genre_queue)
add_artist_genres(user_headers, artist_genre_queue)
if len(features_queue) > 0: if len(features_queue) > 0:
get_audio_features(headers, features_queue)
get_audio_features(user_headers, features_queue)
# }}} clean-up # # }}} clean-up #
update_track_genres(user)
update_track_genres(user_obj)
context = {
'user_id': user_obj.id,
'user_secret': user_obj.secret,
}
return render(request, 'api/logged_in.html', context)
# }}} parse_library # # }}} parse_library #

104
graphs/models.py
View File

@ -1,104 +0,0 @@
from django.db import models
# id's are 22 in length in examples but set to 30 for buffer
MAX_ID = 30
# Genre {{{ #
class Genre(models.Model):
class Meta:
verbose_name = "Genre"
verbose_name_plural = "Genres"
name = models.CharField(primary_key=True, max_length=50)
num_songs = models.PositiveIntegerField()
def __str__(self):
return self.name
# }}} Genre #
# Artist {{{ #
class Artist(models.Model):
class Meta:
verbose_name = "Artist"
verbose_name_plural = "Artists"
artist_id = models.CharField(primary_key=True, max_length=MAX_ID)
# unique since only storing one genre per artist right now
name = models.CharField(unique=True, max_length=50)
genres = models.ManyToManyField(Genre, blank=True)
def __str__(self):
return self.name
# }}} Artist #
# User {{{ #
class User(models.Model):
class Meta:
verbose_name = "User"
verbose_name_plural = "Users"
user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID
user_secret = models.CharField(max_length=50, default='')
def __str__(self):
return self.user_id
# }}} User #
# Track {{{ #
class Track(models.Model):
class Meta:
verbose_name = "Track"
verbose_name_plural = "Tracks"
track_id = models.CharField(primary_key=True, max_length=MAX_ID)
# artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
artists = models.ManyToManyField(Artist, blank=True)
year = models.PositiveSmallIntegerField()
popularity = models.PositiveSmallIntegerField()
runtime = models.PositiveSmallIntegerField()
name = models.CharField(max_length=200)
users = models.ManyToManyField(User, blank=True)
genre = models.ForeignKey(Genre, on_delete=models.CASCADE, blank=True,
null=True)
def __str__(self):
track_str = "{}, genre: {}, artists: [".format(self.name, self.genre)
for artist in self.artists.all():
track_str += "{}, ".format(artist.name)
track_str += "]"
return track_str
# }}} Track #
# AudioFeatures {{{ #
class AudioFeatures(models.Model):
class Meta:
verbose_name = "AudioFeatures"
verbose_name_plural = "AudioFeatures"
track = models.OneToOneField(Track, on_delete=models.CASCADE, primary_key=True,)
acousticness = models.DecimalField(decimal_places=3, max_digits=3)
danceability = models.DecimalField(decimal_places=3, max_digits=3)
energy = models.DecimalField(decimal_places=3, max_digits=3)
instrumentalness = models.DecimalField(decimal_places=3, max_digits=3)
loudness = models.DecimalField(decimal_places=3, max_digits=6)
speechiness = models.DecimalField(decimal_places=3, max_digits=3)
tempo = models.DecimalField(decimal_places=3, max_digits=6)
valence = models.DecimalField(decimal_places=3, max_digits=3)
def __str__(self):
return super(AudioFeatures, self).__str__()
# }}} AudioFeatures #

4
login/models.py
View File

@ -15,8 +15,8 @@ class User(models.Model):
secret = models.CharField(max_length=50, default='') secret = models.CharField(max_length=50, default='')
refresh_token = models.CharField(max_length=TOKEN_LENGTH) refresh_token = models.CharField(max_length=TOKEN_LENGTH)
access_token = models.CharField(max_length=TOKEN_LENGTH) access_token = models.CharField(max_length=TOKEN_LENGTH)
access_obtained_at = models.DateTimeField(auto_now_add=True)
access_obtained_at = models.DateTimeField(auto_now=True)
access_expires_in = models.PositiveIntegerField() access_expires_in = models.PositiveIntegerField()
def __str__(self): def __str__(self):
return self.user_id
return self.id

7
login/templates/login/scan.html
View File

@ -10,13 +10,16 @@
<title>User Spotify Data</title> <title>User Spotify Data</title>
<meta name="description" content=""> <meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css">
<link rel="stylesheet" href="{% static 'css/dark_bg.css' %}"> <link rel="stylesheet" href="{% static 'css/dark_bg.css' %}">
</head> </head>
<body> <body>
<!--[if lt IE 7]> <!--[if lt IE 7]>
<p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="#">upgrade your browser</a> to improve your experience.</p> <p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="#">upgrade your browser</a> to improve your experience.</p>
<![endif]--> <![endif]-->
<p>Logged in as {{ user_id }}</p>
<a href="" class="btn btn-primary">Scan Library</a>
<h1>Logged in as {{ user_id }}</h1>
<a href="{% url "api:scan" user_secret %}" class="btn btn-primary">
Scan Library
</a>
</body> </body>
</html> </html>

35
login/views.py
View File

@ -38,20 +38,6 @@ def generate_random_string(length):
# }}} generate_random_string # # }}} generate_random_string #
# token_expired {{{ #
def token_expired(token_obtained_at, valid_for):
"""Returns True if token expired, False if otherwise
Args:
token_obtained_at: datetime object representing the date and time when the token was obtained
valid_for: the time duration for which the token is valid, in seconds
"""
time_elapsed = (datetime.today() - token_obtained_at).total_seconds()
return time_elapsed >= valid_for
# }}} token_expired #
# index {{{ # # index {{{ #
# Create your views here. # Create your views here.
@ -146,27 +132,6 @@ def create_user(refresh_token, access_token, access_expires_in):
return user_obj return user_obj
# refresh access token {{{ #
"""
token_obtained_at = datetime.strptime(request.session['token_obtained_at'], TIME_FORMAT)
valid_for = int(request.session['valid_for'])
if token_expired(token_obtained_at, valid_for):
req_body = {
'grant_type': 'refresh_token',
'refresh_token': request.session['refresh_token'],
'client_id': os.environ['SPOTIFY_CLIENT_ID'],
'client_secret': os.environ['SPOTIFY_CLIENT_SECRET']
}
refresh_token_response = requests.post('https://accounts.spotify.com/api/token', data=req_body).json()
request.session['access_token'] = refresh_token_response['access_token']
request.session['valid_for'] = refresh_token_response['expires_in']
"""
# }}} refresh access token #
# admin_graphs {{{ # # admin_graphs {{{ #
def admin_graphs(request): def admin_graphs(request):

1
reset_db.sh
View File

@ -11,4 +11,5 @@ rm login/migrations/0* api/migrations/0* graphs/migrations/0*
sudo -u postgres psql -f reset_db.sql sudo -u postgres psql -f reset_db.sql
python manage.py makemigrations python manage.py makemigrations
python manage.py migrate python manage.py migrate
python manage.py runserver
# fi # fi
Write Preview
Loading…
Cancel
Save