Browse Source

Setup API app, can scan library (#47)

Manually merged console logging from loading-page branch.
master
Kevin Mok 6 years ago
parent
commit
c949ecd3cc
  1. 7
      api/models.py
  2. 8
      api/templates/api/logged_in.html
  3. 3
      api/urls.py
  4. 93
      api/utils.py
  5. 80
      api/views.py
  6. 104
      graphs/models.py
  7. 4
      login/models.py
  8. 7
      login/templates/login/scan.html
  9. 35
      login/views.py
  10. 1
      reset_db.sh

7
api/models.py

@ -22,15 +22,14 @@ class Genre(models.Model):
# Artist {{{ #
class Artist(models.Model):
class Meta:
verbose_name = "Artist"
verbose_name_plural = "Artists"
artist_id = models.CharField(primary_key=True, max_length=MAX_ID)
id = models.CharField(primary_key=True, max_length=MAX_ID)
# unique since only storing one genre per artist right now
name = models.CharField(unique=True, max_length=50)
name = models.CharField(max_length=50)
genres = models.ManyToManyField(Genre, blank=True)
def __str__(self):
@ -46,7 +45,7 @@ class Track(models.Model):
verbose_name = "Track"
verbose_name_plural = "Tracks"
track_id = models.CharField(primary_key=True, max_length=MAX_ID)
id = models.CharField(primary_key=True, max_length=MAX_ID)
# artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
artists = models.ManyToManyField(Artist, blank=True)
year = models.PositiveSmallIntegerField()

8
api/templates/api/logged_in.html

@ -5,15 +5,15 @@
<meta charset="UTF-8">
<title>Logged In</title>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
<link rel="stylesheet" href="{% static 'spotifyvis/css/dark_bg.css' %}">
<link rel="stylesheet" href="{% static 'css/dark_bg.css' %}">
</head>
<body>
<h1>{{ user_id }}'s Graphs</h1>
<a class="btn btn-primary" href="{% url "display_audio_features" user_secret %}"
<a class="btn btn-primary" href=""
role="button">Audio Features</a>
<a class="btn btn-primary" href="{% url "display_genre_graph" user_secret %}"
<a class="btn btn-primary" href=""
role="button">Genres</a>
<a class="btn btn-primary" href="{% url "display_artist_graph" user_secret %}" role="button">
<a class="btn btn-primary" href="" role="button">
Artists
</a>
</body>

3
api/urls.py

@ -4,7 +4,8 @@ from .views import *
app_name = 'api'
urlpatterns = [
# path('scan/<str:user_secret>', get_artist_data),
path('scan/<str:user_secret>', parse_library,
name='scan'),
path('user_artists/<str:user_secret>', get_artist_data,
name='get_artist_data'),
path('user_genres/<str:user_secret>', get_genre_data,

93
api/utils.py

@ -2,33 +2,36 @@
import requests
import math
import pprint
import os
import json
from .models import *
from django.db.models import Count, Q, F
from django.http import JsonResponse
from django.core import serializers
import json
from django.utils import timezone
from .models import *
from login.models import User
# }}} imports #
USER_TRACKS_LIMIT = 50
ARTIST_LIMIT = 50
FEATURES_LIMIT = 100
# ARTIST_LIMIT = 25
# FEATURES_LIMIT = 25
console_logging = True
# console_logging = False
artists_genre_processed = 0
features_processed = 0
# update_track_genres {{{ #
def update_track_genres(user):
"""Updates user's tracks with the most common genre associated with the
def update_track_genres(user_obj):
"""Updates user_obj's tracks with the most common genre associated with the
songs' artist(s).
:user: User object who's tracks are being updated.
:user_obj: User object who's tracks are being updated.
:returns: None
"""
user_tracks = Track.objects.filter(users__exact=user)
tracks_processed = 0
user_tracks = Track.objects.filter(users__exact=user_obj)
for track in user_tracks:
# just using this variable to save another call to db
track_artists = track.artists.all()
@ -44,41 +47,46 @@ def update_track_genres(user):
track.genre = most_common_genre if most_common_genre is not None \
else undefined_genre_obj
track.save()
# print(track.name, track.genre)
tracks_processed += 1
if console_logging:
print("Added '{}' as genre for song #{} - '{}'".format(
track.genre,
tracks_processed,
track.name,
))
# }}} update_track_genres #
# save_track_obj {{{ #
def save_track_obj(track_dict, artists, top_genre, user):
def save_track_obj(track_dict, artists, user_obj):
"""Make an entry in the database for this track if it doesn't exist already.
:track_dict: dictionary from the API call containing track information.
:artists: artists of the song, passed in as a list of Artist objects.
:top_genre: top genre associated with this track (see get_top_genre).
:user: User object for which this Track is to be associated with.
:user_obj: User object for which this Track is to be associated with.
:returns: (The created/retrieved Track object, created)
"""
track_query = Track.objects.filter(track_id__exact=track_dict['id'])
track_query = Track.objects.filter(id__exact=track_dict['id'])
if len(track_query) != 0:
return track_query[0], False
else:
new_track = Track.objects.create(
track_id=track_dict['id'],
id=track_dict['id'],
year=track_dict['album']['release_date'].split('-')[0],
popularity=int(track_dict['popularity']),
runtime=int(float(track_dict['duration_ms']) / 1000),
name=track_dict['name'],
# genre=top_genre,
)
# have to add artists and user after saving object since track needs to
# have to add artists and user_obj after saving object since track needs to
# have ID before filling in m2m field
for artist in artists:
new_track.artists.add(artist)
new_track.users.add(user)
new_track.users.add(user_obj)
new_track.save()
return new_track, True
@ -96,13 +104,14 @@ def get_audio_features(headers, track_objs):
:returns: None
"""
track_ids = str.join(",", [track_obj.track_id for track_obj in track_objs])
track_ids = str.join(",", [track_obj.id for track_obj in track_objs])
params = {'ids': track_ids}
features_response = requests.get("https://api.spotify.com/v1/audio-features",
headers=headers,params=params).json()['audio_features']
# pprint.pprint(features_response)
useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", ]
useless_keys = [ "key", "mode", "type", "liveness", "id", "uri",
"track_href", "analysis_url", "time_signature", ]
for i in range(len(track_objs)):
if features_response[i] is not None:
# Data that we don't need
@ -113,6 +122,12 @@ def get_audio_features(headers, track_objs):
setattr(cur_features_obj, key, val)
cur_features_obj.save()
if console_logging:
global features_processed
features_processed += 1
print("Added features for song #{} - {}".format(
features_processed, track_objs[i].name))
# }}} get_audio_features #
def process_artist_genre(genre_name, artist_obj):
@ -145,7 +160,7 @@ def add_artist_genres(headers, artist_objs):
:returns: None
"""
artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs])
artist_ids = str.join(",", [artist_obj.id for artist_obj in artist_objs])
params = {'ids': artist_ids}
artists_response = requests.get('https://api.spotify.com/v1/artists/',
headers=headers, params=params).json()['artists']
@ -157,6 +172,12 @@ def add_artist_genres(headers, artist_objs):
for genre in artists_response[i]['genres']:
process_artist_genre(genre, artist_objs[i])
if console_logging:
global artists_genre_processed
artists_genre_processed += 1
print("Added genres for artist #{} - {}".format(
artists_genre_processed, artist_objs[i].name))
# }}} add_artist_genres #
# get_artists_in_genre {{{ #
@ -192,3 +213,29 @@ def get_artists_in_genre(user, genre, max_songs):
return processed_artist_counts
# }}} get_artists_in_genre #
def get_user_header(user_obj):
"""Returns the authorization string needed to make an API call.
:user_obj: User to return the auth string for.
:returns: the authorization string used for the header in a Spotify API
call.
"""
seconds_elapsed = (timezone.now() -
user_obj.access_obtained_at).total_seconds()
if seconds_elapsed >= user_obj.access_expires_in:
req_body = {
'grant_type': 'refresh_token',
'refresh_token': user_obj.refresh_token,
'client_id': os.environ['SPOTIFY_CLIENT_ID'],
'client_secret': os.environ['SPOTIFY_CLIENT_SECRET']
}
token_response = requests.post('https://accounts.spotify.com/api/token',
data=req_body).json()
user_obj.access_token = token_response['access_token']
user_obj.access_expires_in = token_response['expires_in']
user_obj.save()
return {'Authorization': "Bearer " + user_obj.access_token}

80
api/views.py

@ -3,49 +3,59 @@
import math
import random
import requests
import os
import urllib
import secrets
import pprint
import string
from datetime import datetime
from django.shortcuts import render, redirect
from django.http import JsonResponse
from django.db.models import Count, Q
from .utils import get_artists_in_genre, update_track_genres
from .models import User, Track, AudioFeatures, Artist
from .utils import *
from .models import *
from login.models import User
# }}} imports #
USER_TRACKS_LIMIT = 50
ARTIST_LIMIT = 50
FEATURES_LIMIT = 100
# ARTIST_LIMIT = 25
# FEATURES_LIMIT = 25
TRACKS_TO_QUERY = 200
# parse_library {{{ #
console_logging = True
def parse_library(headers, tracks, user):
"""Scans user's library for certain number of tracks and store the information in a database
# parse_library {{{ #
:headers: For API call.
:tracks: Number of tracks to get from user's library.
:user: a User object representing the user whose library we are parsing
def parse_library(request, user_secret):
"""Scans user's library for num_tracks and store the information in a
database.
:user_secret: secret for User object who's library is being scanned.
:returns: None
"""
# TODO: implement importing entire library with 0 as tracks param
# keeps track of point to get songs from
offset = 0
payload = {'limit': str(USER_TRACKS_LIMIT)}
artist_genre_queue = []
features_queue = []
user_obj = User.objects.get(secret=user_secret)
user_headers = get_user_header(user_obj)
# iterate until hit requested num of tracks
for i in range(0, tracks, USER_TRACKS_LIMIT):
# create this obj so loop runs at least once
saved_tracks_response = [0]
# scan until reach num_tracks or no tracks left if scanning entire library
while (TRACKS_TO_QUERY == 0 or offset < TRACKS_TO_QUERY) and len(saved_tracks_response) > 0:
payload['offset'] = str(offset)
saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks',
headers=headers,
params=payload).json()
headers=user_headers,
params=payload).json()['items']
if console_logging:
tracks_processed = 0
for track_dict in saved_tracks_response['items']:
for track_dict in saved_tracks_response:
# add artists {{{ #
# update artist info before track so that Track object can reference
@ -53,22 +63,20 @@ def parse_library(headers, tracks, user):
track_artists = []
for artist_dict in track_dict['track']['artists']:
artist_obj, artist_created = Artist.objects.get_or_create(
artist_id=artist_dict['id'],
id=artist_dict['id'],
name=artist_dict['name'],)
# only add/tally up artist genres if new
if artist_created:
artist_genre_queue.append(artist_obj)
if len(artist_genre_queue) == ARTIST_LIMIT:
add_artist_genres(headers, artist_genre_queue)
add_artist_genres(user_headers, artist_genre_queue)
artist_genre_queue = []
track_artists.append(artist_obj)
# }}} add artists #
# TODO: fix this, don't need any more
top_genre = ""
track_obj, track_created = save_track_obj(track_dict['track'],
track_artists, top_genre, user)
track_artists, user_obj)
# add audio features {{{ #
@ -77,16 +85,18 @@ def parse_library(headers, tracks, user):
if track_created:
features_queue.append(track_obj)
if len(features_queue) == FEATURES_LIMIT:
get_audio_features(headers, features_queue)
get_audio_features(user_headers, features_queue)
features_queue = []
# }}} add audio features #
# temporary console logging
print("#{}-{}: {} - {}".format(offset + 1,
offset + USER_TRACKS_LIMIT,
track_obj.artists.first(),
track_obj.name))
if console_logging:
tracks_processed += 1
print("Added track #{}: {} - {}".format(
offset + tracks_processed,
track_obj.artists.first(),
track_obj.name,
))
# calculates num_songs with offset + songs retrieved
offset += USER_TRACKS_LIMIT
@ -96,13 +106,19 @@ def parse_library(headers, tracks, user):
# update remaining artists without genres and songs without features if
# there are any
if len(artist_genre_queue) > 0:
add_artist_genres(headers, artist_genre_queue)
add_artist_genres(user_headers, artist_genre_queue)
if len(features_queue) > 0:
get_audio_features(headers, features_queue)
get_audio_features(user_headers, features_queue)
# }}} clean-up #
update_track_genres(user)
update_track_genres(user_obj)
context = {
'user_id': user_obj.id,
'user_secret': user_obj.secret,
}
return render(request, 'api/logged_in.html', context)
# }}} parse_library #

104
graphs/models.py

@ -1,104 +0,0 @@
from django.db import models
# id's are 22 in length in examples but set to 30 for buffer
MAX_ID = 30
# Genre {{{ #
class Genre(models.Model):
class Meta:
verbose_name = "Genre"
verbose_name_plural = "Genres"
name = models.CharField(primary_key=True, max_length=50)
num_songs = models.PositiveIntegerField()
def __str__(self):
return self.name
# }}} Genre #
# Artist {{{ #
class Artist(models.Model):
class Meta:
verbose_name = "Artist"
verbose_name_plural = "Artists"
artist_id = models.CharField(primary_key=True, max_length=MAX_ID)
# unique since only storing one genre per artist right now
name = models.CharField(unique=True, max_length=50)
genres = models.ManyToManyField(Genre, blank=True)
def __str__(self):
return self.name
# }}} Artist #
# User {{{ #
class User(models.Model):
class Meta:
verbose_name = "User"
verbose_name_plural = "Users"
user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID
user_secret = models.CharField(max_length=50, default='')
def __str__(self):
return self.user_id
# }}} User #
# Track {{{ #
class Track(models.Model):
class Meta:
verbose_name = "Track"
verbose_name_plural = "Tracks"
track_id = models.CharField(primary_key=True, max_length=MAX_ID)
# artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
artists = models.ManyToManyField(Artist, blank=True)
year = models.PositiveSmallIntegerField()
popularity = models.PositiveSmallIntegerField()
runtime = models.PositiveSmallIntegerField()
name = models.CharField(max_length=200)
users = models.ManyToManyField(User, blank=True)
genre = models.ForeignKey(Genre, on_delete=models.CASCADE, blank=True,
null=True)
def __str__(self):
track_str = "{}, genre: {}, artists: [".format(self.name, self.genre)
for artist in self.artists.all():
track_str += "{}, ".format(artist.name)
track_str += "]"
return track_str
# }}} Track #
# AudioFeatures {{{ #
class AudioFeatures(models.Model):
class Meta:
verbose_name = "AudioFeatures"
verbose_name_plural = "AudioFeatures"
track = models.OneToOneField(Track, on_delete=models.CASCADE, primary_key=True,)
acousticness = models.DecimalField(decimal_places=3, max_digits=3)
danceability = models.DecimalField(decimal_places=3, max_digits=3)
energy = models.DecimalField(decimal_places=3, max_digits=3)
instrumentalness = models.DecimalField(decimal_places=3, max_digits=3)
loudness = models.DecimalField(decimal_places=3, max_digits=6)
speechiness = models.DecimalField(decimal_places=3, max_digits=3)
tempo = models.DecimalField(decimal_places=3, max_digits=6)
valence = models.DecimalField(decimal_places=3, max_digits=3)
def __str__(self):
return super(AudioFeatures, self).__str__()
# }}} AudioFeatures #

4
login/models.py

@ -15,8 +15,8 @@ class User(models.Model):
secret = models.CharField(max_length=50, default='')
refresh_token = models.CharField(max_length=TOKEN_LENGTH)
access_token = models.CharField(max_length=TOKEN_LENGTH)
access_obtained_at = models.DateTimeField(auto_now_add=True)
access_obtained_at = models.DateTimeField(auto_now=True)
access_expires_in = models.PositiveIntegerField()
def __str__(self):
return self.user_id
return self.id

7
login/templates/login/scan.html

@ -10,13 +10,16 @@
<title>User Spotify Data</title>
<meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css">
<link rel="stylesheet" href="{% static 'css/dark_bg.css' %}">
</head>
<body>
<!--[if lt IE 7]>
<p class="browsehappy">You are using an <strong>outdated</strong> browser. Please <a href="#">upgrade your browser</a> to improve your experience.</p>
<![endif]-->
<p>Logged in as {{ user_id }}</p>
<a href="" class="btn btn-primary">Scan Library</a>
<h1>Logged in as {{ user_id }}</h1>
<a href="{% url "api:scan" user_secret %}" class="btn btn-primary">
Scan Library
</a>
</body>
</html>

35
login/views.py

@ -38,20 +38,6 @@ def generate_random_string(length):
# }}} generate_random_string #
# token_expired {{{ #
def token_expired(token_obtained_at, valid_for):
"""Returns True if token expired, False if otherwise
Args:
token_obtained_at: datetime object representing the date and time when the token was obtained
valid_for: the time duration for which the token is valid, in seconds
"""
time_elapsed = (datetime.today() - token_obtained_at).total_seconds()
return time_elapsed >= valid_for
# }}} token_expired #
# index {{{ #
# Create your views here.
@ -146,27 +132,6 @@ def create_user(refresh_token, access_token, access_expires_in):
return user_obj
# refresh access token {{{ #
"""
token_obtained_at = datetime.strptime(request.session['token_obtained_at'], TIME_FORMAT)
valid_for = int(request.session['valid_for'])
if token_expired(token_obtained_at, valid_for):
req_body = {
'grant_type': 'refresh_token',
'refresh_token': request.session['refresh_token'],
'client_id': os.environ['SPOTIFY_CLIENT_ID'],
'client_secret': os.environ['SPOTIFY_CLIENT_SECRET']
}
refresh_token_response = requests.post('https://accounts.spotify.com/api/token', data=req_body).json()
request.session['access_token'] = refresh_token_response['access_token']
request.session['valid_for'] = refresh_token_response['expires_in']
"""
# }}} refresh access token #
# admin_graphs {{{ #
def admin_graphs(request):

1
reset_db.sh

@ -11,4 +11,5 @@ rm login/migrations/0* api/migrations/0* graphs/migrations/0*
sudo -u postgres psql -f reset_db.sql
python manage.py makemigrations
python manage.py migrate
python manage.py runserver
# fi
Loading…
Cancel
Save