Browse Source

Merge pull request #41 from Kevin-Mok/genre-data

Assign the most common genre to tracks instead of just the first genre for the first artist.
master
Kevin Mok 7 years ago
committed by GitHub
parent
commit
e8dd6881b1
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 8
      recreate-db.txt
  2. 2
      requirements.txt
  3. 22
      spotifyvis/models.py
  4. 8
      spotifyvis/static/spotifyvis/css/dark_bg.css
  5. 6
      spotifyvis/templates/spotifyvis/index.html
  6. 2
      spotifyvis/templates/spotifyvis/test_db.html
  7. 6
      spotifyvis/templates/spotifyvis/user_data.html
  8. 429
      spotifyvis/utils.py
  9. 28
      spotifyvis/views.py

8
recreate-db.txt

@ -0,0 +1,8 @@
# https://stackoverflow.com/a/34576062/8811872
sudo su postgres
psql
drop database spotifyvis;
create database spotifyvis with owner django;
\q
exit

2
requirements.txt

@ -7,7 +7,7 @@ idna==2.6
isort==4.3.4
lazy-object-proxy==1.3.1
mccabe==0.6.1
psycopg2==2.7.4
psycopg2-binary==2.7.4
pylint==1.8.4
pytz==2018.4
requests==2.18.4

22
spotifyvis/models.py

@ -3,6 +3,22 @@ from django.db import models
# id's are 22 in length in examples but set to 30 for buffer
MAX_ID = 30
# Genre {{{ #
class Genre(models.Model):
class Meta:
verbose_name = "Genre"
verbose_name_plural = "Genres"
name = models.CharField(primary_key=True, max_length=50)
num_songs = models.PositiveIntegerField()
def __str__(self):
return self.name
# }}} Genre #
# Artist {{{ #
@ -14,6 +30,7 @@ class Artist(models.Model):
artist_id = models.CharField(primary_key=True, max_length=MAX_ID)
# unique since only storing one genre per artist right now
name = models.CharField(unique=True, max_length=50)
genres = models.ManyToManyField(Genre, blank=True)
def __str__(self):
return self.name
@ -51,7 +68,9 @@ class Track(models.Model):
runtime = models.PositiveSmallIntegerField()
name = models.CharField(max_length=200)
users = models.ManyToManyField(User, blank=True)
genre = models.CharField(max_length=30)
# genre = models.CharField(max_length=30)
genre = models.ForeignKey(Genre, on_delete=models.CASCADE, blank=True,
null=True)
def __str__(self):
return self.name
@ -60,7 +79,6 @@ class Track(models.Model):
# AudioFeatures {{{ #
class AudioFeatures(models.Model):
class Meta:

8
spotifyvis/static/spotifyvis/css/dark_bg.css

@ -0,0 +1,8 @@
body {
background-color: #1e1e1e;
}
h1,p {
color: grey;
}

6
spotifyvis/templates/spotifyvis/index.html

@ -4,6 +4,7 @@
<head>
<title>User Login</title>
<link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css">
<link rel="stylesheet" href="{% static 'spotifyvis/css/dark_bg.css' %}">
<style type="text/css">
.text-overflow {
overflow: hidden;
@ -18,10 +19,9 @@
<body>
<div class="container">
<div id="login">
<h1>This is an example of the Authorization Code flow</h1>
<a href="/login" class="btn btn-primary">Log In (Original)</a>
<h1>spotify-lib-vis</h1>
<a href="/login" class="btn btn-primary">Scan Library</a>
<a href="/test_db" class="btn btn-primary">Test DB</a>
<button id="login-btn">Log In</button>
</div>
<div id="data-container">

2
spotifyvis/templates/spotifyvis/test_db.html

@ -5,6 +5,7 @@
<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8"> <![endif]-->
<!--[if IE 8]> <html class="no-js lt-ie9"> <![endif]-->
<!--[if gt IE 8]><!-->
{% load static %}
<html class="no-js"> <!--<![endif]-->
<head>
<meta charset="utf-8">
@ -12,6 +13,7 @@
<title>Test DB Page</title>
<meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{% static 'spotifyvis/css/dark_bg.css' %}">
</head>
<!-- }}} header -->

6
spotifyvis/templates/spotifyvis/user_data.html

@ -10,16 +10,12 @@
<title>User Spotify Data</title>
<meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="{% static 'spotifyvis/css/dark_bg.css' %}">
</head>
<body>
<!--[if lt IE 7]>
<p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="#">upgrade your browser</a> to improve your experience.</p>
<![endif]-->
<p>Logged in as {{ id }}</p>
<script src="{% static "spotifyvis/scripts/user_data.js" %}"></script>
<script>
sessionStorage.setItem('user_secret', "{{ user_secret }}");
getAudioFeatureData('instrumentalness', sessionStorage.getItem('user_secret'));
</script>
</body>
</html>

429
spotifyvis/utils.py

@ -3,16 +3,21 @@ import requests
import math
import pprint
from .models import Artist, User, Track, AudioFeatures
from django.db.models import Count, Q
from .models import *
from django.db.models import Count, Q, F
from django.http import JsonResponse
from django.core import serializers
import json
# }}} imports #
# parse_library {{{ #
USER_TRACKS_LIMIT = 50
ARTIST_LIMIT = 50
FEATURES_LIMIT = 100
# ARTIST_LIMIT = 25
# FEATURES_LIMIT = 25
# parse_library {{{ #
def parse_library(headers, tracks, user):
"""Scans user's library for certain number of tracks to update library_stats with.
@ -25,59 +30,108 @@ def parse_library(headers, tracks, user):
"""
# TODO: implement importing entire library with 0 as tracks param
# number of tracks to get with each call
limit = 50
# keeps track of point to get songs from
offset = 0
payload = {'limit': str(limit)}
payload = {'limit': str(USER_TRACKS_LIMIT)}
artist_genre_queue = []
features_queue = []
# iterate until hit requested num of tracks
for _ in range(0, tracks, limit):
for i in range(0, tracks, USER_TRACKS_LIMIT):
payload['offset'] = str(offset)
# get current set of tracks
saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', headers=headers, params=payload).json()
saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks',
headers=headers,
params=payload).json()
# TODO: refactor the for loop body into helper function
# iterate through each track
for track_dict in saved_tracks_response['items']:
# add artists {{{ #
# update artist info before track so that Track object can reference
# Artist object
track_artists = []
for artist_dict in track_dict['track']['artists']:
artist_obj, artist_created = Artist.objects.get_or_create(
artist_id=artist_dict['id'],
name=artist_dict['name'],
)
# update_artist_genre(headers, artist_obj)
# get_or_create() returns a tuple (obj, created)
name=artist_dict['name'],)
# only add/tally up artist genres if new
if artist_created:
artist_genre_queue.append(artist_obj)
if len(artist_genre_queue) == ARTIST_LIMIT:
add_artist_genres(headers, artist_genre_queue)
artist_genre_queue = []
track_artists.append(artist_obj)
top_genre = get_top_genre(headers,
track_dict['track']['artists'][0]['id'])
# }}} add artists #
# WIP: get most common genre
top_genre = ""
track_obj, track_created = save_track_obj(track_dict['track'],
track_artists, top_genre, user)
# if a new track is not created, the associated audio feature does not need to be created again
# add audio features {{{ #
# if a new track is not created, the associated audio feature does
# not need to be created again
if track_created:
save_audio_features(headers, track_dict['track']['id'], track_obj)
"""
TODO: Put this logic in another function
# Audio analysis could be empty if not present in Spotify database
if len(audio_features_dict) != 0:
# Track the number of audio analyses for calculating
# audio feature averages and standard deviations on the fly
feature_data_points += 1
for feature, feature_data in audio_features_dict.items():
update_audio_feature_stats(feature, feature_data,
feature_data_points, library_stats)
"""
features_queue.append(track_obj)
if len(features_queue) == FEATURES_LIMIT:
get_audio_features(headers, features_queue)
features_queue = []
# }}} add audio features #
# temporary console logging
print("#{}-{}: {} - {}".format(offset + 1,
offset + USER_TRACKS_LIMIT,
track_obj.artists.first(),
track_obj.name))
# calculates num_songs with offset + songs retrieved
offset += limit
# pprint.pprint(library_stats)
offset += USER_TRACKS_LIMIT
# clean-up {{{ #
# update remaining artists without genres and songs without features if
# there are any
if len(artist_genre_queue) > 0:
add_artist_genres(headers, artist_genre_queue)
if len(features_queue) > 0:
get_audio_features(headers, features_queue)
# }}} clean-up #
update_track_genres(user)
# }}} parse_library #
# update_track_genres {{{ #
def update_track_genres(user):
"""Updates user's tracks with the most common genre associated with the
songs' artist(s).
:user: User object who's tracks are being updated.
:returns: None
"""
user_tracks = Track.objects.filter(users__exact=user)
for track in user_tracks:
# just using this variable to save another call to db
track_artists = track.artists.all()
# set genres to first artist's genres then find intersection with others
shared_genres = track_artists.first().genres.all()
for artist in track_artists:
shared_genres.intersection(artist.genres.all())
most_common_genre = shared_genres.order_by('-num_songs').first()
track.genre = most_common_genre if most_common_genre is not None \
else "undefined"
track.save()
# print(track.name, track.genre)
# }}} update_track_genres #
# save_track_obj {{{ #
def save_track_obj(track_dict, artists, top_genre, user):
@ -87,6 +141,7 @@ def save_track_obj(track_dict, artists, top_genre, user):
:artists: artists of the song, passed in as a list of Artist objects.
:top_genre: top genre associated with this track (see get_top_genre).
:user: User object for which this Track is to be associated with.
:returns: (The created/retrieved Track object, created)
"""
@ -100,7 +155,7 @@ def save_track_obj(track_dict, artists, top_genre, user):
popularity=int(track_dict['popularity']),
runtime=int(float(track_dict['duration_ms']) / 1000),
name=track_dict['name'],
genre=top_genre,
# genre=top_genre,
)
# have to add artists and user after saving object since track needs to
@ -115,282 +170,78 @@ def save_track_obj(track_dict, artists, top_genre, user):
# get_audio_features {{{ #
def save_audio_features(headers, track_id, track):
"""Creates and saves a new AudioFeatures object
def get_audio_features(headers, track_objs):
"""Creates and saves a new AudioFeatures objects for the respective
track_objs. track_objs should contain the API limit for a single call
(FEATURES_LIMIT) for maximum efficiency.
Args:
headers: headers containing the API token
track_id: the id of the soundtrack, needed to query the Spotify API
track: Track object to associate with the new AudioFeatures object
:headers: headers containing the API token
:track_objs: Track objects to associate with the new AudioFeatures object
:returns: None
"""
response = requests.get("https://api.spotify.com/v1/audio-features/{}".format(track_id), headers = headers).json()
if 'error' in response:
return {}
track_ids = str.join(",", [track_obj.track_id for track_obj in track_objs])
params = {'ids': track_ids}
features_response = requests.get("https://api.spotify.com/v1/audio-features",
headers=headers,params=params).json()['audio_features']
# pprint.pprint(features_response)
useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", ]
for i in range(len(track_objs)):
if features_response[i] is not None:
# Data that we don't need
useless_keys = [
"key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature",
]
audio_features_entry = AudioFeatures()
audio_features_entry.track = track
for key, val in response.items():
cur_features_obj = AudioFeatures()
cur_features_obj.track = track_objs[i]
for key, val in features_response[i].items():
if key not in useless_keys:
setattr(audio_features_entry, key, val)
audio_features_entry.save()
setattr(cur_features_obj, key, val)
cur_features_obj.save()
# }}} get_audio_features #
# update_std_dev {{{ #
def update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size):
"""Calculates the standard deviation for a sample without storing all data points
Args:
cur_mean: the current mean for N = (sample_size - 1)
cur_std_dev: the current standard deviation for N = (sample_size - 1)
new_data_point: a new data point
sample_size: sample size including the new data point
Returns:
(new_mean, new_std_dev)
"""
# This is an implementation of Welford's method
# http://jonisalonen.com/2013/deriving-welfords-method-for-computing-variance/
new_mean = ((sample_size - 1) * cur_mean + new_data_point) / sample_size
delta_variance = (new_data_point - new_mean) * (new_data_point - cur_mean)
new_std_dev = math.sqrt(
(math.pow(cur_std_dev, 2) * (sample_size - 2) + delta_variance) / (
sample_size - 1
))
return new_mean, new_std_dev
# }}} update_std_dev #
# update_audio_feature_stats {{{ #
def update_audio_feature_stats(feature, new_data_point, sample_size, library_stats):
"""Updates the audio feature statistics in library_stats
Args:
feature: the audio feature to be updated (string)
new_data_point: new data to update the stats with
sample_size: sample size including the new data point
library_stats Dictionary containing the data mined from user's Spotify library
Returns:
None
"""
# first time the feature is considered
if sample_size < 2:
library_stats['audio_features'][feature] = {
"average": new_data_point,
"std_dev": 0,
}
else:
cur_mean = library_stats['audio_features'][feature]['average']
cur_std_dev = library_stats['audio_features'][feature]['std_dev']
new_mean, new_std_dev = update_std_dev(cur_mean, cur_std_dev, new_data_point, sample_size)
library_stats['audio_features'][feature] = {
"average": new_mean,
"std_dev": new_std_dev
}
# }}} update_audio_feature_stats #
# increase_nested_key {{{ #
def increase_nested_key(top_key, nested_key, library_stats, amount=1):
"""Increases count for the value of library_stats[top_key][nested_key]. Checks if nested_key exists already and takes
appropriate action.
:top_key: First key of library_stats.
:nested_key: Key in top_key's dict for which we want to increase value of.
:library_stats: Dictionary containing the data mined from user's Spotify library
def process_artist_genre(genre_name, artist_obj):
"""Increase count for correspoding Genre object to genre_name and add that
Genre to artist_obj.
:genre_name: Name of genre.
:artist_obj: Artist object to add Genre object to.
:returns: None
"""
if nested_key not in library_stats[top_key]:
library_stats[top_key][nested_key] = amount
else:
library_stats[top_key][nested_key] += amount
# }}} increase_nested_key #
# increase_artist_count {{{ #
def increase_artist_count(headers, artist_name, artist_id, library_stats):
"""Increases count for artist in library_stats and stores the artist_id.
:headers: For making the API call.
:artist_name: Artist to increase count for.
:artist_id: The Spotify ID for the artist.
:library_stats: Dictionary containing the data mined from user's Spotify library
:returns: None
"""
if artist_name not in library_stats['artists']:
library_stats['artists'][artist_name] = {}
library_stats['artists'][artist_name]['count'] = 1
library_stats['artists'][artist_name]['id'] = artist_id
else:
library_stats['artists'][artist_name]['count'] += 1
# }}} increase_artist_count #
# update_popularity_stats {{{ #
def update_popularity_stats(new_data_point, library_stats, sample_size):
"""Updates the popularity statistics in library_stats
Args:
new_data_point: new data to update the popularity stats with
library_stats: Dictionary containing data mined from user's Spotify library
sample_size: The sample size including the new data
Returns:
None
"""
if sample_size < 2:
library_stats['popularity'] = {
"average": new_data_point,
"std_dev": 0,
}
else :
cur_mean_popularity = library_stats['popularity']['average']
cur_popularity_stdev = library_stats['popularity']['std_dev']
new_mean, new_std_dev = update_std_dev(
cur_mean_popularity, cur_popularity_stdev, new_data_point, sample_size)
library_stats['popularity'] = {
"average": new_mean,
"std_dev": new_std_dev,
}
# }}} update_popularity_stats #
# get_track_info {{{ #
def get_track_info(track_dict, library_stats, sample_size):
"""Get all the info from the track_dict directly returned by the API call in parse_library.
:track_dict: Dict returned from the API call containing the track info.
:library_stats: Dictionary containing the data mined from user's Spotify library
:sample_size: The sample size so far including this track
:returns: None
"""
# popularity
update_popularity_stats(track_dict['popularity'], library_stats, sample_size)
# year
year_released = track_dict['album']['release_date'].split('-')[0]
increase_nested_key('year_released', year_released, library_stats)
# runtime
library_stats['total_runtime'] += float(track_dict['duration_ms']) / (1000 * 60)
# }}} get_track_info #
# update_artist_genre {{{ #
def update_artist_genre(headers, artist_obj):
"""Updates the top genre for an artist by querying the Spotify API
:headers: For making the API call.
:artist_obj: the Artist object whose genre field will be updated
:returns: None
"""
artist_response = requests.get('https://api.spotify.com/v1/artists/' + artist_obj.artist_id, headers=headers).json()
# update genre for artist in database with top genre
if len(artist_response['genres']) > 0:
artist_obj.genre = artist_response['genres'][0]
genre_obj, created = Genre.objects.get_or_create(name=genre_name,
defaults={'num_songs':1})
if not created:
genre_obj.num_songs = F('num_songs') + 1
genre_obj.save()
artist_obj.genres.add(genre_obj)
artist_obj.save()
# }}} #
# add_artist_genres {{{ #
# get_top_genre {{{ #
def get_top_genre(headers, top_artist_id):
"""Updates the top genre for a track by querying the Spotify API
def add_artist_genres(headers, artist_objs):
"""Adds genres to artist_objs and increases the count the respective Genre
object. artist_objs should contain the API limit for a single call
(ARTIST_LIMIT) for maximum efficiency.
:headers: For making the API call.
:top_artist: The first artist's (listed in the track) Spotify ID.
:artist_objs: List of Artist objects for which to add/tally up genres for.
:returns: The first genre listed for the top_artist.
:returns: None
"""
artist_response = requests.get('https://api.spotify.com/v1/artists/' +
top_artist_id, headers=headers).json()
if len(artist_response['genres']) > 0:
return artist_response['genres'][0]
artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs])
params = {'ids': artist_ids}
artists_response = requests.get('https://api.spotify.com/v1/artists/',
headers=headers, params=params).json()['artists']
# pprint.pprint(artists_response)
for i in range(len(artist_objs)):
if len(artists_response[i]['genres']) == 0:
process_artist_genre("undefined", artist_objs[i])
else:
return "undefined"
# }}} #
for genre in artists_response[i]['genres']:
process_artist_genre(genre, artist_objs[i])
# process_library_stats {{{ #
def process_library_stats(library_stats):
"""Processes library_stats into format more suitable for D3 consumption
Args:
library_stats: Dictionary containing the data mined from user's Spotify library
Returns:
A new dictionary that contains the data in library_stats, in a format more suitable for D3 consumption
"""
processed_library_stats = {}
for key in library_stats:
if key == 'artists' or key == 'genres' or key == 'year_released':
for inner_key in library_stats[key]:
if key not in processed_library_stats:
processed_library_stats[key] = []
processed_item_key = '' # identifier key for each dict in the list
count = 0
if 'artist' in key:
processed_item_key = 'name'
count = library_stats[key][inner_key]['count']
elif 'genre' in key:
processed_item_key = 'genre'
count = library_stats[key][inner_key]
else:
processed_item_key = 'year'
count = library_stats[key][inner_key]
processed_library_stats[key].append({
processed_item_key: inner_key,
"count": count
})
elif key == 'audio_features':
for audio_feature in library_stats[key]:
if 'audio_features' not in processed_library_stats:
processed_library_stats['audio_features'] = []
processed_library_stats['audio_features'].append({
'feature': audio_feature,
'average': library_stats[key][audio_feature]['average'],
'std_dev': library_stats[key][audio_feature]['std_dev']
})
# TODO: Not sure about final form for 'popularity'
# elif key == 'popularity':
# processed_library_stats[key] = []
# processed_library_stats[key].append({
# })
elif key == 'num_songs' or key == 'total_runtime' or key == 'popularity':
processed_library_stats[key] = library_stats[key]
return processed_library_stats
# }}} process_library_stats #
# }}} add_artist_genres #
# get_artists_in_genre {{{ #
@ -413,6 +264,8 @@ def get_artists_in_genre(user, genre, max_songs):
processed_artist_counts = {}
songs_added = 0
for artist in artist_counts:
# hacky way to not have total count overflow due to there being multiple
# artists on a track
if songs_added + artist.num_songs <= max_songs:
processed_artist_counts[artist.name] = artist.num_songs
songs_added += artist.num_songs

28
spotifyvis/views.py

@ -13,13 +13,13 @@ from datetime import datetime
from django.shortcuts import render, redirect
from django.http import HttpResponse, HttpResponseBadRequest, JsonResponse
from django.db.models import Count, Q
from .utils import parse_library, process_library_stats, get_artists_in_genre
from .utils import parse_library, get_artists_in_genre, update_track_genres
from .models import User, Track, AudioFeatures, Artist
# }}} imports #
TIME_FORMAT = '%Y-%m-%d-%H-%M-%S'
TRACKS_TO_QUERY = 100
TRACKS_TO_QUERY = 200
# generate_random_string {{{ #
@ -42,7 +42,6 @@ def generate_random_string(length):
# token_expired {{{ #
def token_expired(token_obtained_at, valid_for):
"""Returns True if token expired, False if otherwise
@ -65,8 +64,8 @@ def index(request):
# login {{{ #
# uses Authorization Code flow
def login(request):
# use a randomly generated state string to prevent cross-site request forgery attacks
state_str = generate_random_string(16)
request.session['state_string'] = state_str
@ -119,6 +118,9 @@ def callback(request):
# user_data {{{ #
def user_data(request):
# get user token {{{ #
token_obtained_at = datetime.strptime(request.session['token_obtained_at'], TIME_FORMAT)
valid_for = int(request.session['valid_for'])
@ -134,14 +136,18 @@ def user_data(request):
request.session['access_token'] = refresh_token_response['access_token']
request.session['valid_for'] = refresh_token_response['expires_in']
# }}} get user token #
auth_token_str = "Bearer " + request.session['access_token']
headers = {
'Authorization': auth_token_str
}
user_data_response = requests.get('https://api.spotify.com/v1/me', headers = headers).json()
request.session['user_id'] = user_data_response['id'] # store the user_id so it may be used to create model
# request.session['user_name'] = user_data_response['display_name']
# store the user_id so it may be used to create model
request.session['user_id'] = user_data_response['id']
# create user obj {{{ #
try:
user = User.objects.get(user_id=user_data_response['id'])
@ -149,6 +155,8 @@ def user_data(request):
user = User(user_id=user_data_response['id'], user_secret=generate_random_string(30))
user.save()
# }}} create user obj #
context = {
'id': user_data_response['id'],
'user_secret': user.user_secret,
@ -165,10 +173,12 @@ def test_db(request):
"""TODO
"""
user_id = "polarbier"
user_obj = User.objects.get(user_id=user_id)
# user_id = "35kxo00qqo9pd1comj6ylxjq7"
context = {
'user_secret': User.objects.get(user_id=user_id).user_secret,
'user_secret': user_obj.user_secret,
}
update_track_genres(user_obj)
return render(request, 'spotifyvis/test_db.html', context)
# }}} test_db #
@ -203,8 +213,11 @@ def get_audio_feature_data(request, audio_feature, client_secret):
'data_points': [],
}
for track in user_tracks:
try:
audio_feature_obj = AudioFeatures.objects.get(track=track)
response_payload['data_points'].append(getattr(audio_feature_obj, audio_feature))
except AudioFeatures.DoesNotExist:
continue
return JsonResponse(response_payload)
# }}} get_audio_feature_data #
@ -224,6 +237,7 @@ def get_genre_data(request, user_secret):
for genre_dict in genre_counts:
genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre'],
genre_dict['num_songs'])
print("*** Genre Breakdown ***")
pprint.pprint(list(genre_counts))
return JsonResponse(data=list(genre_counts), safe=False)

Loading…
Cancel
Save