
# CHANGE THIS IF YOU NEED/WANT TOO

# pandas / numpy
import pandas as pd
import numpy as np
rng = np.random.default_rng() # for random number generation

# plotting
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib import rcParams
from matplotlib.lines import Line2D

# datetime operations
from datetime import datetime

# ttest and euclidean distance
from scipy.stats import ttest_ind
from scipy.spatial.distance import seuclidean

# linear fit using statsmodels
import statsmodels.api as sm
import statsmodels.formula.api as smf

# good ole sklearn
from sklearn.metrics import euclidean_distances, r2_score
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import train_test_split

# displaying markdown strings
from IPython.display import display, Markdown, Latex


# settings
rcParams['figure.dpi']= 100
pd.set_option('display.max_rows', 3000)
sns.set(style="whitegrid")


# data folder
#DATA_FOLDER = 'data/'
DATA_FOLDER = r'/Users/geoffraymarie/EPFL/ADA/ADA2021/Homework/02 - Regression Observational Studies Stats and Supervised Learning/data/'


df = pd.read_csv(DATA_FOLDER + 'pitchfork.csv.gz', compression='gzip', 
                 parse_dates=['reviewdate'], dtype={'releaseyear': int}
                )


shape_before = df.shape[0]
df.drop_duplicates(subset = ['artist','album'], keep = 'first', inplace = True)
print('After removing the duplicate {} reviews, the dataframe has now {} rows.'\
      .format(shape_before - df.shape[0], df.shape[0]))

After removing the duplicate 47 reviews, the dataframe has now 16738 rows.


fig, ax1 = plt.subplots()

yearmin, yearmax = df.releaseyear.min(), df.releaseyear.max()

sns.histplot(x = 'releaseyear', stat = 'count', data = df, ax = ax1, 
             bins = [k for k in range(yearmin, yearmax + 1)])

ax1.minorticks_on()
ax1.set_yscale('log')
ax1.tick_params(axis='x', labelrotation = 30)

ax2 = ax1.twinx()
ax2.plot(df.groupby('releaseyear').score.agg('mean'), color = 'red', marker = '+', lw = 0.7)
ax2.set_ylabel('Average score of albums');
ax2.grid(None)


plt.title('Distribution of album release years and the average score of albums per year');


df.describe().loc[['mean', '50%', 'min', 'max']]


aberrant_num_columns = ['danceability', 'energy', 'speechiness', 'acousticness', 'instrumentalness', 
                        'liveness', 'valence', 'tempo']

to_drop = df[df[aberrant_num_columns].lt(0.0).any(axis = 1)].shape[0]
df = df[~df[aberrant_num_columns].lt(0.0).any(axis = 1)]
print('We removed {} rows with irregular values.'.format(to_drop))

We removed 0 rows with irregular values.


df.describe().loc[['mean', '50%', 'min', 'max']]


# should we ask about pitch class ?

#I think we should ask about the keys. Because they should be intergers, 
#but it's the case only for 17% of the data
#df.key[df.key.isin([k for k in range(-1,12)])].shape[0]/df.shape[0]
df.key.hist(bins = 100)

<AxesSubplot:>


fig, ax = plt.subplots(4,3,figsize= (14,14))#, sharex = 'row', sharey = 'row')

# list of names of numerical columns
num_cols = df.describe().loc[['mean', '50%', 'min', 'max']].columns.values
for i, elt in enumerate(num_cols):
    sbplt = ax[i//3, i%3]
    sbplt.hist(df[elt])#, bins = 100)
    sbplt.set_title(elt)
    sbplt.set_yscale('log')
    
fig.tight_layout(pad=0.5)
fig.subplots_adjust(top=0.93)

plt.suptitle('Distribution of all numerical features');


to_modif = (df.genre == 'none') | (df.genre.isna())
df.loc[to_modif, 'genre'] = 'Other'

print(f"There were {to_modif.sum()} genre values replaces by 'Other'.")

There were 11 genre values replaces by 'Other'.


# list of categorical columns
cat_cols = df.columns.difference(num_cols)

for cat in cat_cols:
    n = df[cat].unique().size 
    if n <= 10:
        print(f'"{cat}" column has {n} distinct values : \n{", ".join(df[cat].unique())}.')
    else :
        print(f'"{cat}" column has {n} distinct values.')
    print()

"album" column has 16176 distinct values.

"artist" column has 7890 distinct values.

"genre" column has 10 distinct values : 
Electronic, Folk/Country, Rock, Rap, Global, Experimental, Metal, Pop/R&B, Jazz, Other.

"recordlabel" column has 3030 distinct values.

"reviewauthor" column has 554 distinct values.

"reviewdate" column has 4876 distinct values.


sns.histplot(x = 'genre', stat = 'count', data = df[cat_cols])
plt.xticks(rotation=70);
plt.grid(axis = 'x', b = None)

#Y/N ?
#plt.yscale('log')
plt.title('Repartiton of the different genres');


df_test = df.copy()
count_art = df.artist.value_counts()
count_art.name = 'count_art'
df_test = df_test.merge(count_art,right_index=True, left_on='artist', how='inner')
sns.regplot(data=df_test, y = 'score', x='count_art', line_kws={'color': 'red'} )
plt.title('Relation between number of albums of the artist and score of the album.');


onehot = pd.get_dummies(df.genre)
onehot.columns = [c+'_onehot' for c in onehot.columns]
onehot.head()


def numpy_helper(df, cols):
    '''Return the numpy array of selected cols in df.'''
    return df[cols].values


X = numpy_helper(onehot, onehot.columns)
y = numpy_helper(df, 'score')
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=123)


class Baseline():
    def __init__(self):
        self.avg = None
        
    def fit(self, X, y):
        self.avg = np.mean(y)
    
    def predict(self, X):
        return self.avg * np.ones((X.shape[0],))


baseline = Baseline()
baseline.fit(X_train, y_train)


r2_score(y_test, baseline.predict(y_test))

-1.0839329994682956e-05


model = LinearRegression()

# data conversion to numpy array and random splitting
df_onehot = pd.merge(df, onehot, left_index=True, right_index=True)
cols = [ "releaseyear", "key", "acousticness", "danceability", 
        "energy", "instrumentalness", "liveness", "loudness", 
        "speechiness", "valence", "tempo"] + list(onehot.columns)
X,y = numpy_helper(df_onehot, cols), numpy_helper(df_onehot, 'score')
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3, random_state=123)

#model training
model.fit(X_train, y_train)

print(f"The R^2 of the linear regression on the random split is {r2_score(y_test, model.predict(X_test)):.4f}.")

The R^2 of the linear regression on the random split is 0.0387.


model = LinearRegression()

#copy to silence setting on copy warnings arising later
df_train = df_onehot[df_onehot.releaseyear < 2000].copy()
df_test = df_onehot[df_onehot.releaseyear >= 2003].copy()

X_train,y_train = numpy_helper(df_train, cols), numpy_helper(df_train, 'score')
X_test ,y_test  = numpy_helper(df_test, cols) , numpy_helper(df_test, 'score')

#model training
model.fit(X_train, y_train)

print(f"The R^2 of the linear regression on the longitudinal split is {r2_score(y_test, model.predict(X_test)):.4f}.")

The R^2 of the linear regression on the longitudinal split is -0.2823.


df_train.loc[:,'Period'] = 'Before 2000'
df_test.loc[:,'Period'] = '2003 or after'
df_plot = pd.concat((df_train, df_test), axis=0)
sns.histplot(df_plot, x='score', hue='Period', common_norm=False, stat='density')
plt.title('Evolution of the distribution of scores before 200 and after 2003.');


y_pred = model.predict(X_test)
sns.histplot(x=y_test - y_pred)
plt.title('Residuals of the 3.2 model on the test set')
plt.xlabel("$Y- Y'$");


#bootstrap CI
N = 10_000
proba = np.zeros(N)
n = y_test.size

for i in range(N):
    idx = rng.integers(0,n,n) # n integers between 0 and n : indices of the samples
    y_sample = y_test[idx]
    X_sample = X_test[idx]
    y_pred = model.predict(X_sample)
    err = np.where(np.abs(y_pred - y_sample) > 2, 1, 0) #1 if |res| > 2, else 0
    proba[i] = err.mean()


sns.histplot(x=proba, binwidth=5e-4)
plt.xlabel('Probability')
plt.title(f'Probability of prediction being off by more than 2 scores points,\n $N = {N}$')
plt.xticks(rotation = 45);

Column	Description
`artist`	The name of the artist who created the album being reviewed.
`album`	The name of the album being reviewed.
`recordlabel`	The name of the record label(s) who published the album.
`releaseyear`	The year that the album was released.
`score`	The score given to the album by the reviewer on a scale of 0.0 to 10.0.
`reviewauthor`	The name of the author who reviewed the album.
`genre`	The genre assigned to the album by Pitchfork.
`reviewdate`	The date that the review was published.
`key`	The estimated overall musical key of the track. Integers map to pitches using standard Pitch Class notation (e.g., 0 = C, 2 = D, and so on)
`acousticness`	A confidence measure from 0.0 to 1.0 of whether an album is acoustic. 1.0 represents high confidencethat the album is acoustic.
`danceability`	How suitable an album is for dancing based on a combination of musical elements including tempo, rhythm stability, beat strength, and overall regularity. A value of 1.0 is most danceable.
`energy`	A perceptual measure of intensity and activity, from 0.0 to 1.0, where 1.0 represents high energy. Metal is often high energy.
`instrumentalness`	Predicts whether an album contains no vocals, from 0.0 to 1.0. The closer to 1.0, the more likely the album contains no vocals.
`liveness`	Detects the presence of an audience, from 0.0 to 1.0. Scores greater than 0.8 indicate a strong likelihood the album is live.
`loudness`	The overall loudness of the album in decibels (dB).
`speechiness`	Measures the presence of spoken words in an album on a scale from 0.0 to 1.0. Scores higher than 0.66 indicate an album made entirely of spoken words, while scores below 0.33 indicate music and other non-speech-like elements.
`valence`	A measure from 0.0 to 1.0 describing the musical positiveness conveyed by an album, where values closer to 1.0 indicate more positive sounds.
`tempo`	The overall estimated tempo of an album in beats per minute (BPM).

	score	releaseyear	danceability	energy	key	loudness	speechiness	acousticness	instrumentalness	liveness	valence	tempo
mean	7.048596	2009.346338	0.512334	0.601276	5.216501	-9.283268	0.090742	0.301914	0.274748	0.196402	0.405268	120.326487
50%	7.300000	2010.000000	0.511348	0.624722	5.230769	-8.444263	0.056665	0.228844	0.149363	0.174261	0.406288	120.397346
min	0.000000	1957.000000	-1.000000	-1.000000	-1.000000	-51.728750	-1.000000	-1.000000	-1.000000	-1.000000	-1.000000	-1.000000
max	10.000000	2019.000000	0.974000	0.999000	11.000000	4.078000	0.958000	0.996000	0.982000	0.978000	0.971000	215.972000

	score	releaseyear	danceability	energy	key	loudness	speechiness	acousticness	instrumentalness	liveness	valence	tempo
mean	7.048536	2009.345965	0.512967	0.601951	5.219034	-9.286912	0.091208	0.302476	0.275303	0.196910	0.405853	120.377216
50%	7.300000	2010.000000	0.511364	0.624823	5.230769	-8.446428	0.056681	0.229000	0.149641	0.174304	0.406300	120.406490
min	0.000000	1957.000000	0.038667	0.000126	0.000000	-51.728750	0.008644	0.000001	0.000000	0.015300	0.000010	23.983333
max	10.000000	2019.000000	0.974000	0.999000	11.000000	4.078000	0.958000	0.996000	0.982000	0.978000	0.971000	215.972000

	Electronic_onehot	Folk/Country_onehot	Rap_onehot	Rock_onehot
0	1	0	0	0
1	0	1	0	0
2	1	0	0	0
3	0	0	0	1
4	0	0	1	0

Homework 2 (HW2)¶

Important Dates¶

Some rules¶

Context¶

The data¶

Part 1: Will this album be a hit?¶

Task 2 (Pre-processing and pipeline code — 12 pts)¶