Merge branch 'apps' into loading-page

api/admin.py

@@ -0,0 +1,8 @@
+from django.contrib import admin
+from .models import Track, Artist, AudioFeatures, User
+
+# Register your models here.
+admin.site.register(Track)
+admin.site.register(Artist)
+admin.site.register(AudioFeatures)
+admin.site.register(User)

api/apps.py

@@ -0,0 +1,4 @@
+from django.apps import AppConfig
+
+class ApiConfig(AppConfig):
+    name = 'api'

api/models.py

@@ -0,0 +1,88 @@
+from django.db import models
+from login.models import User
+
+# id's are 22 in length in examples but set to 30 for buffer
+MAX_ID = 30
+
+#  Genre {{{ # 
+
+class Genre(models.Model):
+            
+    class Meta:
+        verbose_name = "Genre"
+        verbose_name_plural = "Genres"
+
+    name = models.CharField(primary_key=True, max_length=50)
+    num_songs = models.PositiveIntegerField()
+
+    def __str__(self):
+        return self.name
+
+#  }}} Genre # 
+
+#  Artist {{{ # 
+
+class Artist(models.Model):
+    class Meta:
+        verbose_name = "Artist"
+        verbose_name_plural = "Artists"
+
+    id = models.CharField(primary_key=True, max_length=MAX_ID)
+    name = models.CharField(max_length=50)
+    genres = models.ManyToManyField(Genre, blank=True)
+
+    def __str__(self):
+        return self.name
+
+#  }}} Artist # 
+
+#  Track {{{ # 
+
+class Track(models.Model):
+    
+    class Meta:
+        verbose_name = "Track"
+        verbose_name_plural = "Tracks"
+
+    id = models.CharField(primary_key=True, max_length=MAX_ID)
+    #  artist = models.ForeignKey(Artist, on_delete=models.CASCADE)
+    artists = models.ManyToManyField(Artist, blank=True)
+    year = models.PositiveSmallIntegerField()
+    popularity = models.PositiveSmallIntegerField()
+    runtime = models.PositiveSmallIntegerField()
+    name = models.CharField(max_length=200)
+    users = models.ManyToManyField(User, blank=True)
+    genre = models.ForeignKey(Genre, on_delete=models.CASCADE, blank=True,
+            null=True)
+
+    def __str__(self):
+        track_str = "{}, genre: {}, artists: [".format(self.name, self.genre)
+        for artist in self.artists.all():
+            track_str += "{}, ".format(artist.name)
+        track_str += "]"
+        return track_str
+
+#  }}} Track # 
+
+#  AudioFeatures {{{ # 
+
+class AudioFeatures(models.Model):
+        
+    class Meta:
+        verbose_name = "AudioFeatures"
+        verbose_name_plural = "AudioFeatures"
+
+    track = models.OneToOneField(Track, on_delete=models.CASCADE, primary_key=True,)
+    acousticness = models.DecimalField(decimal_places=3, max_digits=3)
+    danceability = models.DecimalField(decimal_places=3, max_digits=3)
+    energy = models.DecimalField(decimal_places=3, max_digits=3)
+    instrumentalness = models.DecimalField(decimal_places=3, max_digits=3)
+    loudness = models.DecimalField(decimal_places=3, max_digits=6)
+    speechiness = models.DecimalField(decimal_places=3, max_digits=3)
+    tempo = models.DecimalField(decimal_places=3, max_digits=6)
+    valence = models.DecimalField(decimal_places=3, max_digits=3)
+
+    def __str__(self):
+        return super(AudioFeatures, self).__str__()
+
+#  }}} AudioFeatures #

api/urls.py

@@ -0,0 +1,15 @@
+from django.urls import path, include
+
+from .views import *
+
+app_name = 'api'
+urlpatterns = [
+    path('scan/<str:user_secret>', parse_library,
+        name='scan'),
+    path('user_artists/<str:user_secret>', get_artist_data,
+        name='get_artist_data'),
+    path('user_genres/<str:user_secret>', get_genre_data,
+        name='get_genre_data'),
+    path('audio_features/<str:audio_feature>/<str:user_secret>',
+        get_audio_feature_data, name='get_audio_feature_data'),
+]

api/utils.py

@@ -0,0 +1,248 @@
+#  imports {{{ # 
+import requests
+import math
+import pprint
+import os
+import json
+
+from django.db.models import Count, Q, F
+from django.http import JsonResponse
+from django.core import serializers
+from django.utils import timezone
+from .models import *
+from login.models import User
+
+#  }}} imports # 
+
+console_logging = True
+#  console_logging = False
+artists_genre_processed = 0
+features_processed = 0
+
+#  update_track_genres {{{ # 
+
+def update_track_genres(user_obj):
+    """Updates user_obj's tracks with the most common genre associated with the
+    songs' artist(s).
+
+    :user_obj: User object who's tracks are being updated.
+
+    :returns: None
+
+    """
+    tracks_processed = 0
+    user_tracks = Track.objects.filter(users__exact=user_obj)
+    for track in user_tracks:
+        # just using this variable to save another call to db
+        track_artists = track.artists.all()
+        # set genres to first artist's genres then find intersection with others
+        shared_genres = track_artists.first().genres.all()
+        for artist in track_artists:
+            shared_genres = shared_genres.intersection(artist.genres.all())
+        shared_genres = shared_genres.order_by('-num_songs')
+
+        undefined_genre_obj = Genre.objects.get(name="undefined")
+        most_common_genre = shared_genres.first() if shared_genres.first() is \
+                not undefined_genre_obj else shared_genres[1]
+        track.genre = most_common_genre if most_common_genre is not None \
+                else undefined_genre_obj
+        track.save()
+        tracks_processed += 1
+
+        if console_logging:
+            print("Added '{}' as genre for song #{} - '{}'".format(
+                track.genre, 
+                tracks_processed,
+                track.name,
+                ))
+
+#  }}}  update_track_genres # 
+
+#  save_track_obj {{{ # 
+
+def save_track_obj(track_dict, artists, user_obj):
+    """Make an entry in the database for this track if it doesn't exist already.
+
+    :track_dict: dictionary from the API call containing track information.
+    :artists: artists of the song, passed in as a list of Artist objects.
+    :user_obj: User object for which this Track is to be associated with.
+
+    :returns: (The created/retrieved Track object, created) 
+
+    """
+    track_query = Track.objects.filter(id__exact=track_dict['id'])
+    if len(track_query) != 0:
+        return track_query[0], False
+    else:
+        new_track = Track.objects.create(
+            id=track_dict['id'],
+            year=track_dict['album']['release_date'].split('-')[0],
+            popularity=int(track_dict['popularity']),
+            runtime=int(float(track_dict['duration_ms']) / 1000),
+            name=track_dict['name'],
+            )
+
+        # have to add artists and user_obj after saving object since track needs to
+        # have ID before filling in m2m field
+        for artist in artists:
+            new_track.artists.add(artist)
+        new_track.users.add(user_obj)
+        new_track.save()
+        return new_track, True
+
+#  }}} save_track_obj # 
+
+#  get_audio_features {{{ # 
+
+def get_audio_features(headers, track_objs):
+    """Creates and saves a new AudioFeatures objects for the respective
+    track_objs. track_objs should contain the API limit for a single call
+    (FEATURES_LIMIT) for maximum efficiency.
+
+    :headers: headers containing the API token
+    :track_objs: Track objects to associate with the new AudioFeatures object
+        
+    :returns: None
+    """
+    track_ids = str.join(",", [track_obj.id for track_obj in track_objs])
+    params = {'ids': track_ids}
+    features_response = requests.get("https://api.spotify.com/v1/audio-features",
+            headers=headers,
+            params={'ids': track_ids}
+            ).json()['audio_features']
+    #  pprint.pprint(features_response)
+
+    useless_keys = [ "key", "mode", "type", "liveness", "id", "uri",
+            "track_href", "analysis_url", "time_signature", ]
+    for i in range(len(track_objs)):
+        if features_response[i] is not None:
+            # Data that we don't need
+            cur_features_obj = AudioFeatures()
+            cur_features_obj.track = track_objs[i]
+            for key, val in features_response[i].items():
+                if key not in useless_keys:
+                    setattr(cur_features_obj, key, val)
+            cur_features_obj.save()
+
+        if console_logging:
+            global features_processed 
+            features_processed += 1
+            print("Added features for song #{} - {}".format(
+                features_processed, track_objs[i].name))
+
+#  }}} get_audio_features # 
+
+#  process_artist_genre {{{ # 
+
+def process_artist_genre(genre_name, artist_obj):
+    """Increase count for correspoding Genre object to genre_name and add that
+    Genre to artist_obj. 
+
+    :genre_name: Name of genre.
+    :artist_obj: Artist object to add Genre object to.
+    :returns: None
+
+    """
+    genre_obj, created = Genre.objects.get_or_create(name=genre_name,
+            defaults={'num_songs':1})
+    if not created:
+        genre_obj.num_songs = F('num_songs') + 1
+        genre_obj.save()
+    artist_obj.genres.add(genre_obj)
+    artist_obj.save()
+
+#  }}} process_artist_genre # 
+
+#  add_artist_genres {{{ # 
+
+def add_artist_genres(headers, artist_objs):
+    """Adds genres to artist_objs and increases the count the respective Genre
+    object. artist_objs should contain the API limit for a single call
+    (ARTIST_LIMIT) for maximum efficiency.
+
+    :headers: For making the API call.
+    :artist_objs: List of Artist objects for which to add/tally up genres for.
+
+    :returns: None
+
+    """
+    artist_ids = str.join(",", [artist_obj.id for artist_obj in artist_objs])
+    params = {'ids': artist_ids}
+    artists_response = requests.get('https://api.spotify.com/v1/artists/',
+            headers=headers, 
+            params={'ids': artist_ids},
+            ).json()['artists']
+    for i in range(len(artist_objs)):
+        if len(artists_response[i]['genres']) == 0:
+            process_artist_genre("undefined", artist_objs[i])
+        else:
+            for genre in artists_response[i]['genres']:
+                process_artist_genre(genre, artist_objs[i])
+
+        if console_logging:
+            global artists_genre_processed 
+            artists_genre_processed += 1
+            print("Added genres for artist #{} - {}".format(
+                artists_genre_processed, artist_objs[i].name))
+
+#  }}}  add_artist_genres # 
+
+#  get_artists_in_genre {{{ # 
+
+def get_artists_in_genre(user, genre, max_songs):
+    """Return count of artists in genre.
+
+    :user: User object to return data for.
+    :genre: genre to count artists for.
+    :max_songs: max total songs to include to prevent overflow due to having
+    multiple artists on each track.
+
+    :returns: dict of artists in the genre along with the number of songs they
+    have. 
+    """
+    genre_obj = Genre.objects.get(name=genre)
+    artist_counts = (Artist.objects.filter(track__users=user)
+            .filter(genres=genre_obj) 
+            .annotate(num_songs=Count('track', distinct=True))
+            .order_by('-num_songs')
+            )
+    processed_artist_counts = {}
+    songs_added = 0
+    for artist in artist_counts:
+        # hacky way to not have total count overflow due to there being multiple
+        # artists on a track
+        if songs_added + artist.num_songs <= max_songs:
+            processed_artist_counts[artist.name] = artist.num_songs
+            songs_added += artist.num_songs
+    #  processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} for artist in artist_counts]
+    #  processed_artist_counts = {artist.name: artist.num_songs for artist in artist_counts}
+    #  pprint.pprint(processed_artist_counts)
+    return processed_artist_counts
+
+#  }}} get_artists_in_genre # 
+
+def get_user_header(user_obj):
+    """Returns the authorization string needed to make an API call.
+
+    :user_obj: User to return the auth string for.
+    :returns: the authorization string used for the header in a Spotify API
+    call.
+
+    """
+    seconds_elapsed = (timezone.now() -
+            user_obj.access_obtained_at).total_seconds()
+    if seconds_elapsed >= user_obj.access_expires_in:
+        req_body = {
+            'grant_type': 'refresh_token',
+            'refresh_token': user_obj.refresh_token,
+            'client_id': os.environ['SPOTIFY_CLIENT_ID'],
+            'client_secret': os.environ['SPOTIFY_CLIENT_SECRET']
+        }
+        
+        token_response = requests.post('https://accounts.spotify.com/api/token',
+                data=req_body).json()
+        user_obj.access_token = token_response['access_token']
+        user_obj.access_expires_in = token_response['expires_in']
+        user_obj.save()
+
+    return {'Authorization': "Bearer " + user_obj.access_token}

api/views.py

@@ -0,0 +1,186 @@
+#  imports {{{ # 
+
+import math
+import random
+import requests
+import urllib
+import secrets
+import pprint
+import string
+
+from django.shortcuts import render, redirect
+from django.http import JsonResponse
+from django.db.models import Count, Q
+from .utils import *
+from .models import *
+from login.models import User
+from login.utils import get_user_context
+
+#  }}} imports # 
+
+USER_TRACKS_LIMIT = 50
+ARTIST_LIMIT = 50
+FEATURES_LIMIT = 100
+#  ARTIST_LIMIT = 25
+#  FEATURES_LIMIT = 25
+TRACKS_TO_QUERY = 100
+
+console_logging = True
+
+#  parse_library {{{ # 
+
+def parse_library(request, user_secret):
+    """Scans user's library for num_tracks and store the information in a
+    database.
+
+    :user_secret: secret for User object who's library is being scanned.
+    :returns: None
+    """
+
+    offset = 0
+    payload = {'limit': str(USER_TRACKS_LIMIT)}
+    artist_genre_queue = []
+    features_queue = []
+    user_obj = User.objects.get(secret=user_secret)
+    user_headers = get_user_header(user_obj)
+
+    # create this obj so loop runs at least once
+    saved_tracks_response = [0]
+    # scan until reach num_tracks or no tracks left if scanning entire library
+    while (TRACKS_TO_QUERY == 0 or offset < TRACKS_TO_QUERY) and len(saved_tracks_response) > 0:
+        payload['offset'] = str(offset)
+        saved_tracks_response = requests.get('https://api.spotify.com/v1/me/tracks', 
+                headers=user_headers,
+                params=payload).json()['items']
+
+        if console_logging:
+            tracks_processed = 0
+
+        for track_dict in saved_tracks_response:
+            #  add artists {{{ # 
+            
+            # update artist info before track so that Track object can reference
+            # Artist object
+            track_artists = []
+            for artist_dict in track_dict['track']['artists']:
+                artist_obj, artist_created = Artist.objects.get_or_create(
+                        id=artist_dict['id'],
+                        name=artist_dict['name'],)
+                # only add/tally up artist genres if new
+                if artist_created:
+                    artist_genre_queue.append(artist_obj)
+                    if len(artist_genre_queue) == ARTIST_LIMIT:
+                        add_artist_genres(user_headers, artist_genre_queue)
+                        artist_genre_queue = []
+                track_artists.append(artist_obj)
+            
+            #  }}} add artists # 
+            
+            track_obj, track_created = save_track_obj(track_dict['track'], 
+                    track_artists, user_obj)
+
+            #  add audio features {{{ # 
+            
+            # if a new track is not created, the associated audio feature does
+            # not need to be created again
+            if track_created:
+                features_queue.append(track_obj)
+                if len(features_queue) == FEATURES_LIMIT:
+                    get_audio_features(user_headers, features_queue)
+                    features_queue = []
+            
+            #  }}} add audio features # 
+
+            if console_logging:
+                tracks_processed += 1
+                print("Added track #{}: {} - {}".format(
+                    offset + tracks_processed,
+                    track_obj.artists.first(), 
+                    track_obj.name,
+                    ))
+
+        # calculates num_songs with offset + songs retrieved
+        offset += USER_TRACKS_LIMIT
+
+    #  clean-up {{{ # 
+    
+    # update remaining artists without genres and songs without features if
+    # there are any
+    if len(artist_genre_queue) > 0:
+        add_artist_genres(user_headers, artist_genre_queue)
+    if len(features_queue) > 0:
+        get_audio_features(user_headers, features_queue)
+    
+    #  }}} clean-up # 
+
+    update_track_genres(user_obj)
+
+    return render(request, 'graphs/logged_in.html', get_user_context(user_obj))
+
+#  }}} parse_library # 
+
+#  get_artist_data {{{ # 
+
+def get_artist_data(request, user_secret):
+    """Returns artist data as a JSON serialized list of dictionaries
+    The (key, value) pairs are (artist name, song count for said artist)
+
+    :param request: the HTTP request
+    :param user_secret: the user secret used for identification
+    :return: a JsonResponse
+    """
+    user = User.objects.get(secret=user_secret)
+    artist_counts = Artist.objects.annotate(num_songs=Count('track',
+        filter=Q(track__users=user)))
+    processed_artist_counts = [{'name': artist.name, 'num_songs': artist.num_songs} 
+            for artist in artist_counts]
+    pprint.pprint(processed_artist_counts)
+    return JsonResponse(data=processed_artist_counts, safe=False) 
+
+#  }}} get_artist_data # 
+
+#  get_audio_feature_data {{{ # 
+
+def get_audio_feature_data(request, audio_feature, user_secret):
+    """Returns all data points for a given audio feature
+
+    Args:
+        request: the HTTP request
+        audio_feature: The audio feature to be queried
+        user_secret: client secret, used to identify the user
+    """
+    user = User.objects.get(secret=user_secret)
+    user_tracks = Track.objects.filter(users=user)
+    response_payload = {
+        'data_points': [],
+    }
+    for track in user_tracks:
+        try:
+            audio_feature_obj = AudioFeatures.objects.get(track=track)
+            response_payload['data_points'].append(getattr(audio_feature_obj, audio_feature))
+        except AudioFeatures.DoesNotExist:
+            continue
+    return JsonResponse(response_payload)
+
+#  }}} get_audio_feature_data # 
+
+#  get_genre_data {{{ # 
+
+def get_genre_data(request, user_secret):
+    """Return genre data needed to create the graph user.
+    TODO
+    """
+    user = User.objects.get(secret=user_secret)
+    genre_counts = (Track.objects.filter(users__exact=user)
+            .values('genre')
+            .order_by('genre')
+            .annotate(num_songs=Count('genre'))
+            )
+    for genre_dict in genre_counts:
+        genre_dict['artists'] = get_artists_in_genre(user, genre_dict['genre'],
+                genre_dict['num_songs'])
+    print("*** Genre Breakdown ***")
+    pprint.pprint(list(genre_counts))
+    return JsonResponse(data=list(genre_counts), safe=False) 
+
+#  }}} get_genre_data  #