Split spotifyvis code into different apps (#47)

Server is able to start, but none of the apps are linked together yet.
2018-06-29 04:15:08 -04:00
parent 8851c5ce25
commit 8b1344d453
34 changed files with 319 additions and 538 deletions
--- a/api/init.py
+++ b/api/init.py
--- a/api/admin.py
+++ b/api/admin.py
@@ -0,0 +1,8 @@
+from django.contrib import admin
+from .models import Track, Artist, AudioFeatures, User
+
+# Register your models here.
+admin.site.register(Track)
+admin.site.register(Artist)
+admin.site.register(AudioFeatures)
+admin.site.register(User)
--- a/api/apps.py
+++ b/api/apps.py
@@ -0,0 +1,4 @@
+from django.apps import AppConfig
+
+class ApiConfig(AppConfig):
+    name = 'api'
--- a/api/models.py
+++ b/api/models.py
@@ -0,0 +1,104 @@
+from django.db import models
+
+# id's are 22 in length in examples but set to 30 for buffer
+MAX_ID = 30
+
+#  Genre {{{ # 
+
+class Genre(models.Model):
+            
+    class Meta:
+        verbose_name = "Genre"
+        verbose_name_plural = "Genres"
+
+    name = models.CharField(primary_key=True, max_length=50)
+    num_songs = models.PositiveIntegerField()
+
+    def __str__(self):
+        return self.name
+
+#  }}} Genre # 
+
+#  Artist {{{ # 
+
+
+class Artist(models.Model):
+    class Meta:
+        verbose_name = "Artist"
+        verbose_name_plural = "Artists"
+
+    artist_id = models.CharField(primary_key=True, max_length=MAX_ID)
+    # unique since only storing one genre per artist right now
+    name = models.CharField(unique=True, max_length=50)
+    genres = models.ManyToManyField(Genre, blank=True)
+
+    def __str__(self):
+        return self.name
+
+#  }}} Artist # 
+
+#  User {{{ # 
+
+class User(models.Model):
+    class Meta:
+        verbose_name = "User"
+        verbose_name_plural = "Users"
+
+    user_id = models.CharField(primary_key=True, max_length=MAX_ID) # the user's Spotify ID
+    user_secret = models.CharField(max_length=50, default='')
+
+    def __str__(self):
+        return self.user_id
+
+#  }}} User # 
+
+#  Track {{{ # 
+
+class Track(models.Model):
+    
+    class Meta:
+        verbose_name = "Track"
+        verbose_name_plural = "Tracks"
+
+    track_id = models.CharField(primary_key=True, max_length=MAX_ID)
+    #  artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
+    artists = models.ManyToManyField(Artist, blank=True)
+    year = models.PositiveSmallIntegerField()
+    popularity = models.PositiveSmallIntegerField()
+    runtime = models.PositiveSmallIntegerField()
+    name = models.CharField(max_length=200)
+    users = models.ManyToManyField(User, blank=True)
+    genre = models.ForeignKey(Genre, on_delete=models.CASCADE, blank=True,
+            null=True)
+
+    def __str__(self):
+        track_str = "{}, genre: {}, artists: [".format(self.name, self.genre)
+        for artist in self.artists.all():
+            track_str += "{}, ".format(artist.name)
+        track_str += "]"
+        return track_str
+
+#  }}} Track # 
+
+#  AudioFeatures {{{ # 
+
+class AudioFeatures(models.Model):
+        
+    class Meta:
+        verbose_name = "AudioFeatures"
+        verbose_name_plural = "AudioFeatures"
+
+    track = models.OneToOneField(Track, on_delete=models.CASCADE, primary_key=True,)
+    acousticness = models.DecimalField(decimal_places=3, max_digits=3)
+    danceability = models.DecimalField(decimal_places=3, max_digits=3)
+    energy = models.DecimalField(decimal_places=3, max_digits=3)
+    instrumentalness = models.DecimalField(decimal_places=3, max_digits=3)
+    loudness = models.DecimalField(decimal_places=3, max_digits=6)
+    speechiness = models.DecimalField(decimal_places=3, max_digits=3)
+    tempo = models.DecimalField(decimal_places=3, max_digits=6)
+    valence = models.DecimalField(decimal_places=3, max_digits=3)
+
+    def __str__(self):
+        return super(AudioFeatures, self).__str__()
+
+#  }}} AudioFeatures #
--- a/api/templates/api/logged_in.html
+++ b/api/templates/api/logged_in.html
@@ -0,0 +1,20 @@
+<!DOCTYPE html>
+{% load static %}
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Logged In</title>
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
+    <link rel="stylesheet" href="{% static 'spotifyvis/css/dark_bg.css' %}">
+</head>
+<body>
+	<h1>{{ user_id }}'s Graphs</h1>
+	<a class="btn btn-primary" href="{% url "display_audio_features" user_secret %}"
+		role="button">Audio Features</a>
+    <a class="btn btn-primary" href="{% url "display_genre_graph" user_secret %}"
+		role="button">Genres</a>
+    <a class="btn btn-primary" href="{% url "display_artist_graph" user_secret %}" role="button">
+        Artists
+    </a>
+</body>
+</html>
--- a/api/urls.py
+++ b/api/urls.py
@@ -0,0 +1,12 @@
+from django.urls import path, include
+
+from .views import *
+
+urlpatterns = [
+    path('user_artists/<str:user_secret>', get_artist_data,
+        name='get_artist_data'),
+    path('user_genres/<str:user_secret>', get_genre_data,
+        name='get_genre_data'),
+    path('audio_features/<str:audio_feature>/<str:user_secret>',
+        get_audio_feature_data, name='get_audio_feature_data'),
+]
--- a/api/utils.py
+++ b/api/utils.py
@@ -0,0 +1,281 @@
+#  imports {{{ # 
+import requests
+import math
+import pprint
+
+from .models import *
+from django.db.models import Count, Q, F
+from django.http import JsonResponse
+from django.core import serializers
+import json
+
+#  }}} imports # 
+
+USER_TRACKS_LIMIT = 50
+ARTIST_LIMIT = 50
+FEATURES_LIMIT = 100
+#  ARTIST_LIMIT = 25
+#  FEATURES_LIMIT = 25
+
+#  parse_library {{{ # 
+
+def parse_library(headers, tracks, user):
+    """Scans user's library for certain number of tracks and store the information in a database
+
+    :headers: For API call.
+    :tracks: Number of tracks to get from user's library.
+    :user: a User object representing the user whose library we are parsing
+
+    :returns: None
+
+    """
+    #  TODO: implement importing entire library with 0 as tracks param
+    # keeps track of point to get songs from
+    offset = 0
+    payload = {'limit': str(USER_TRACKS_LIMIT)}
+    artist_genre_queue = []
+    features_queue = []
+
+    # iterate until hit requested num of tracks
+    for i in range(0, tracks, USER_TRACKS_LIMIT):
+        payload['offset'] = str(offset)
+        saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', 
+                headers=headers,
+                params=payload).json()
+
+        for track_dict in saved_tracks_response['items']:
+            #  add artists {{{ # 
+            
+            # update artist info before track so that Track object can reference
+            # Artist object
+            track_artists = []
+            for artist_dict in track_dict['track']['artists']:
+                artist_obj, artist_created = Artist.objects.get_or_create(
+                        artist_id=artist_dict['id'],
+                        name=artist_dict['name'],)
+                # only add/tally up artist genres if new
+                if artist_created:
+                    artist_genre_queue.append(artist_obj)
+                    if len(artist_genre_queue) == ARTIST_LIMIT:
+                        add_artist_genres(headers, artist_genre_queue)
+                        artist_genre_queue = []
+                track_artists.append(artist_obj)
+            
+            #  }}} add artists # 
+            
+            # TODO: fix this, don't need any more
+            top_genre = ""
+            track_obj, track_created = save_track_obj(track_dict['track'], 
+                    track_artists, top_genre, user)
+
+            #  add audio features {{{ # 
+            
+            # if a new track is not created, the associated audio feature does
+            # not need to be created again
+            if track_created:
+                features_queue.append(track_obj)
+                if len(features_queue) == FEATURES_LIMIT:
+                    get_audio_features(headers, features_queue)
+                    features_queue = []
+            
+            #  }}} add audio features # 
+
+            # temporary console logging
+            print("#{}-{}: {} - {}".format(offset + 1,
+                offset + USER_TRACKS_LIMIT, 
+                track_obj.artists.first(), 
+                track_obj.name))
+
+        # calculates num_songs with offset + songs retrieved
+        offset += USER_TRACKS_LIMIT
+
+    #  clean-up {{{ # 
+    
+    # update remaining artists without genres and songs without features if
+    # there are any
+    if len(artist_genre_queue) > 0:
+        add_artist_genres(headers, artist_genre_queue)
+    if len(features_queue) > 0:
+        get_audio_features(headers, features_queue)
+    
+    #  }}} clean-up # 
+
+    update_track_genres(user)
+
+#  }}} parse_library # 
+
+#  update_track_genres {{{ # 
+
+def update_track_genres(user):
+    """Updates user's tracks with the most common genre associated with the
+    songs' artist(s).
+
+    :user: User object who's tracks are being updated.
+
+    :returns: None
+
+    """
+    user_tracks = Track.objects.filter(users__exact=user)
+    for track in user_tracks:
+        # just using this variable to save another call to db
+        track_artists = track.artists.all()
+        # set genres to first artist's genres then find intersection with others
+        shared_genres = track_artists.first().genres.all()
+        for artist in track_artists:
+            shared_genres = shared_genres.intersection(artist.genres.all())
+        shared_genres = shared_genres.order_by('-num_songs')
+
+        undefined_genre_obj = Genre.objects.get(name="undefined")
+        most_common_genre = shared_genres.first() if shared_genres.first() is \
+                not undefined_genre_obj else shared_genres[1]
+        track.genre = most_common_genre if most_common_genre is not None \
+                else undefined_genre_obj
+        track.save()
+        #  print(track.name, track.genre)
+
+#  }}}  update_track_genres # 
+
+#  save_track_obj {{{ # 
+
+def save_track_obj(track_dict, artists, top_genre, user):
+    """Make an entry in the database for this track if it doesn't exist already.
+
+    :track_dict: dictionary from the API call containing track information.
+    :artists: artists of the song, passed in as a list of Artist objects.
+    :top_genre: top genre associated with this track (see get_top_genre).
+    :user: User object for which this Track is to be associated with.
+
+    :returns: (The created/retrieved Track object, created) 
+
+    """
+    track_query = Track.objects.filter(track_id__exact=track_dict['id'])
+    if len(track_query) != 0:
+        return track_query[0], False
+    else:
+        new_track = Track.objects.create(
+            track_id=track_dict['id'],
+            year=track_dict['album']['release_date'].split('-')[0],
+            popularity=int(track_dict['popularity']),
+            runtime=int(float(track_dict['duration_ms']) / 1000),
+            name=track_dict['name'],
+            #  genre=top_genre,
+            )
+
+        # have to add artists and user after saving object since track needs to
+        # have ID before filling in m2m field
+        for artist in artists:
+            new_track.artists.add(artist)
+        new_track.users.add(user)
+        new_track.save()
+        return new_track, True
+
+#  }}} save_track_obj # 
+
+#  get_audio_features {{{ # 
+
+def get_audio_features(headers, track_objs):
+    """Creates and saves a new AudioFeatures objects for the respective
+    track_objs. track_objs should contain the API limit for a single call
+    (FEATURES_LIMIT) for maximum efficiency.
+
+    :headers: headers containing the API token
+    :track_objs: Track objects to associate with the new AudioFeatures object
+        
+    :returns: None
+    """
+    track_ids = str.join(",", [track_obj.track_id for track_obj in track_objs])
+    params = {'ids': track_ids}
+    features_response = requests.get("https://api.spotify.com/v1/audio-features",
+            headers=headers,params=params).json()['audio_features']
+    #  pprint.pprint(features_response)
+
+    useless_keys = [ "key", "mode", "type", "liveness", "id", "uri", "track_href", "analysis_url", "time_signature", ]
+    for i in range(len(track_objs)):
+        if features_response[i] is not None:
+            # Data that we don't need
+            cur_features_obj = AudioFeatures()
+            cur_features_obj.track = track_objs[i]
+            for key, val in features_response[i].items():
+                if key not in useless_keys:
+                    setattr(cur_features_obj, key, val)
+            cur_features_obj.save()
+
+#  }}} get_audio_features # 
+
+def process_artist_genre(genre_name, artist_obj):
+    """Increase count for correspoding Genre object to genre_name and add that
+    Genre to artist_obj. 
+
+    :genre_name: Name of genre.
+    :artist_obj: Artist object to add Genre object to.
+    :returns: None
+
+    """
+    genre_obj, created = Genre.objects.get_or_create(name=genre_name,
+            defaults={'num_songs':1})
+    if not created:
+        genre_obj.num_songs = F('num_songs') + 1
+        genre_obj.save()
+    artist_obj.genres.add(genre_obj)
+    artist_obj.save()
+
+#  add_artist_genres {{{ # 
+
+def add_artist_genres(headers, artist_objs):
+    """Adds genres to artist_objs and increases the count the respective Genre
+    object. artist_objs should contain the API limit for a single call
+    (ARTIST_LIMIT) for maximum efficiency.
+
+    :headers: For making the API call.
+    :artist_objs: List of Artist objects for which to add/tally up genres for.
+
+    :returns: None
+
+    """
+    artist_ids = str.join(",", [artist_obj.artist_id for artist_obj in artist_objs])
+    params = {'ids': artist_ids}
+    artists_response = requests.get('https://api.spotify.com/v1/artists/',
+            headers=headers, params=params).json()['artists']
+    #  pprint.pprint(artists_response)
+    for i in range(len(artist_objs)):
+        if len(artists_response[i]['genres']) == 0:
+            process_artist_genre("undefined", artist_objs[i])
+        else:
+            for genre in artists_response[i]['genres']:
+                process_artist_genre(genre, artist_objs[i])
+
+#  }}}  add_artist_genres # 
+
+#  get_artists_in_genre {{{ # 
+
+def get_artists_in_genre(user, genre, max_songs):
+    """Return count of artists in genre.
+
+    :user: User object to return data for.
+    :genre: genre to count artists for.
+    :max_songs: max total songs to include to prevent overflow due to having
+    multiple artists on each track.
+
+    :returns: dict of artists in the genre along with the number of songs they
+    have. 
+    """
+    genre_obj = Genre.objects.get(name=genre)
+    artist_counts = (Artist.objects.filter(track__users=user)
+            .filter(genres=genre_obj) 
+            .annotate(num_songs=Count('track', distinct=True))
+            .order_by('-num_songs')
+            )
+    processed_artist_counts = {}
+    songs_added = 0
+    for artist in artist_counts:
+        # hacky way to not have total count overflow due to there being multiple
+        # artists on a track
+        if songs_added + artist.num_songs <= max_songs:
+            processed_artist_counts[artist.name] = artist.num_songs
+            songs_added += artist.num_songs
+    #  processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts]
+    #  processed_artist_counts = {artist.name: artist.num_songs for artist in artist_counts}
+    #  pprint.pprint(processed_artist_counts)
+    return processed_artist_counts
+
+#  }}} get_artists_in_genre # 
--- a/api/views.py
+++ b/api/views.py
@@ -0,0 +1,86 @@
+#  imports {{{ # 
+
+import math
+import random
+import requests
+import os
+import urllib
+import secrets
+import pprint
+import string
+from datetime import datetime
+
+from django.http import JsonResponse
+from django.db.models import Count, Q
+from .utils import parse_library, get_artists_in_genre, update_track_genres
+from .models import User, Track, AudioFeatures, Artist 
+
+#  }}} imports # 
+
+TRACKS_TO_QUERY = 200
+
+#  get_artist_data {{{ # 
+
+
+def get_artist_data(request, user_secret):
+    """Returns artist data as a JSON serialized list of dictionaries
+    The (key, value) pairs are (artist name, song count for said artist)
+
+    :param request: the HTTP request
+    :param user_secret: the user secret used for identification
+    :return: a JsonResponse
+    """
+    user = User.objects.get(user_secret=user_secret)
+    artist_counts = Artist.objects.annotate(num_songs=Count('track',
+                                            filter=Q(track__users=user)))
+    processed_artist_counts = [{'name': artist.name,
+                                'num_songs': artist.num_songs} for artist in artist_counts]
+    return JsonResponse(data=processed_artist_counts, safe=False) 
+
+#  }}} get_artist_data # 
+
+#  get_audio_feature_data {{{ # 
+
+def get_audio_feature_data(request, audio_feature, user_secret):
+    """Returns all data points for a given audio feature
+
+    Args:
+        request: the HTTP request
+        audio_feature: The audio feature to be queried
+        user_secret: client secret, used to identify the user
+    """
+    user = User.objects.get(user_secret=user_secret)
+    user_tracks = Track.objects.filter(users=user)
+    response_payload = {
+        'data_points': [],
+    }
+    for track in user_tracks:
+        try:
+            audio_feature_obj = AudioFeatures.objects.get(track=track)
+            response_payload['data_points'].append(getattr(audio_feature_obj, audio_feature))
+        except AudioFeatures.DoesNotExist:
+            continue
+    return JsonResponse(response_payload)
+
+#  }}} get_audio_feature_data # 
+
+#  get_genre_data {{{ # 
+
+def get_genre_data(request, user_secret):
+    """Return genre data needed to create the graph user.
+    TODO
+    """
+    user = User.objects.get(user_secret=user_secret)
+    genre_counts = (Track.objects.filter(users__exact=user)
+            .values('genre')
+            .order_by('genre')
+            .annotate(num_songs=Count('genre'))
+            )
+    for genre_dict in genre_counts:
+        genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre'],
+                genre_dict['num_songs'])
+    print("*** Genre Breakdown ***")
+    pprint.pprint(list(genre_counts))
+    return JsonResponse(data=list(genre_counts), safe=False) 
+
+#  }}} get_genre_data  #