endlesspint

import numpy as np
import pandas as pd
import csv
import codecs
from bs4 import BeautifulSoup
import re
import os
import socket
import urllib
import time
import requests

gtrend1 = pd.read_csv('timelines_brownTruckJoin/multiTimeline (1).csv', skiprows=2, index_col=0)
gtrend1.columns = [col.split(': ')[0].lower().replace(' ', '_') for col in gtrend1.columns]
gtrend1.head()

gtrend2 = pd.read_csv('timelines_brownTruckJoin/multiTimeline (2).csv', skiprows=2, index_col=0)
gtrend2.columns = [col.split(': ')[0].lower().replace(' ', '_') for col in gtrend2.columns]
gtrend2.head()

def import_gtrend(file_path):
    gtrend = pd.read_csv(file_path, skiprows=2, index_col=0)
    gtrend.columns = [col.split(': ')[0].lower().replace(' ', '_') for col in gtrend.columns]
    return gtrend

import_gtrend('timelines_brownTruckJoin/multiTimeline (3).csv').head()

pd.concat([gtrend1.brown_truck_brewery, gtrend2.brown_truck_brewery, 
           np.isfinite(gtrend1.brown_truck_brewery/gtrend2.brown_truck_brewery)
          ], axis=1).head()

np.mean((gtrend1.brown_truck_brewery/gtrend2.brown_truck_brewery)[np.isfinite(gtrend1.brown_truck_brewery/gtrend2.brown_truck_brewery)])

def gtrend_norm(df_1, df_2, col='brown_truck_brewery'):
    norm_mult = np.mean((df_1.brown_truck_brewery/df_2.brown_truck_brewery)[np.isfinite(df_1.brown_truck_brewery/df_2.brown_truck_brewery)])
    return norm_mult

gtrend_norm(gtrend1, gtrend2)

def df_norm(df_2, norm_mult):
    return df_2.iloc[:,1:] * norm_mult

df_norm(gtrend2, 0.4036).head()

gtrend1.join(gtrend2.iloc[:,1:] * gtrend_norm(gtrend1, gtrend2)).head()

os.listdir('timelines_brownTruckJoin/')

dir_trends = 'timelines_brownTruckJoin/'
node_file = os.listdir(dir_trends)[0]

def gtrend_join(file_dir = dir_trends, node_file = node_file):
    
    node_file = import_gtrend(file_dir + node_file)
    
    for i in os.listdir(file_dir)[1:]:
        
        temp_file = import_gtrend(file_dir + i)
        node_file = node_file.join(df_norm(temp_file, gtrend_norm(node_file, temp_file)),
                                   lsuffix = '', rsuffix = '_y')
#         node_file = pd.merge(node_file, df_norm(temp_file, gtrend_norm(node_file, temp_file)), 
#                              left_index=True, right_index=True, suffixes=('', '_y'))
    
    return node_file


combined_trends = gtrend_join()
print combined_trends.shape
combined_trends.head()

from collections import Counter

cnt = Counter()

for i in os.listdir(dir_trends):
    temp_file = import_gtrend(dir_trends + i)
    for col in temp_file.columns:
        cnt[col] += 1
        
[brw for brw in cnt if cnt[brw] > 1]

none_dupes = [brw for brw in combined_trends.columns.tolist() if '_y' not in brw]
combined_trends = combined_trends[none_dupes]
print combined_trends.shape
combined_trends.head()

# combined_trends.to_csv('timelines_combined/combined_trends.csv', sep=';')

import matplotlib.pyplot as plt
import matplotlib
matplotlib.style.use('ggplot')
%matplotlib inline

pre_gabf16 = combined_trends.index <= '2016-10-05'
post_gabf16 = combined_trends.index >= '2016-10-09'

pre_post_avgs = combined_trends[pre_gabf16].mean().to_frame(name='pre_gabf16').join(combined_trends[post_gabf16].mean().to_frame(name='post_gabf16'))
pre_post_avgs.head()

pre_post_avgs['dif_gabf16_norm'] = pre_post_avgs.post_gabf16/(pre_post_avgs.pre_gabf16 + 0.000001)
pre_post_avgs.head()

pre_post_avgs.dif_gabf16_norm.describe()

plt.figure(figsize=(12, 6))
plt.hist(pre_post_avgs.dif_gabf16_norm)
plt.title('Search Engagement Ratio - Post GABF Medal (2016)')
plt.ylabel('Brewery Count')

plt.boxplot(pre_post_avgs.dif_gabf16_norm)

# 1/6 of breweries have NO Google search data
np.sum(combined_trends.mean(axis=0) == 0.0)

mean_idx_trends = combined_trends.divide(combined_trends.mean(axis=0))
mean_idx_trends.head()

mean_idx_trends.plot(legend=False, figsize=(16, 8))
plt.title('Brewery Search Hits (indexed)')
plt.ylabel('relative to own avg')

mean_idx_trends[pre_gabf16].max().sort_values(ascending=False)[:10]

mean_idx_trends[post_gabf16].max().sort_values(ascending=False)[:10]

(mean_idx_trends[pre_gabf16].sum(axis=1)).describe()

(mean_idx_trends[post_gabf16].sum(axis=1)).describe()

gabf16_type = pd.read_excel('comp_breweries/gabf16_type.xlsx', index_col=0)

print gabf16_type.shape
gabf16_type.tail()

gabf16_new = gabf16_type[gabf16_type.RETAIN == True]

print gabf16_new.shape
gabf16_new.tail()

gabf16_new[['name_Gtrends', 'state', 'brewery_type', 'medal_cnt']].head()

gabf16_eda = pd.merge(gabf16_new[['brewery', 'name_Gtrends', 'state', 'brewery_type', 'medal_cnt']], pre_post_avgs, 
                      left_on='name_Gtrends', right_index=True).drop_duplicates()

print gabf16_eda.shape
gabf16_eda.head()

gabf16_eda['multiple_wins'] = 0
gabf16_eda.loc[gabf16_eda.medal_cnt>1, 'multiple_wins'] = 1

gabf16_eda.multiple_wins.value_counts()

gabf16_eda.pivot_table(index='brewery_type', columns='multiple_wins', values='brewery', aggfunc='count')

gabf16_eda.pivot_table(index='brewery_type', columns='multiple_wins', values='dif_gabf16_norm', aggfunc='mean')

gabf16_eda[(gabf16_eda.multiple_wins==0) & (gabf16_eda.brewery_type=='Brewpub')].nlargest(3,'dif_gabf16_norm')

gabf16_eda[(gabf16_eda.multiple_wins==0) & (gabf16_eda.brewery_type=='Micro')].nlargest(3,'dif_gabf16_norm')

gabf16_eda[(gabf16_eda.multiple_wins==0) & (gabf16_eda.brewery_type=='Regional')].nlargest(3,'dif_gabf16_norm')

gabf16_eda[(gabf16_eda.multiple_wins==1) & (gabf16_eda.brewery_type=='Brewpub')].nlargest(3,'dif_gabf16_norm')

gabf16_eda[(gabf16_eda.multiple_wins==1) & (gabf16_eda.brewery_type=='Micro')].nlargest(3,'dif_gabf16_norm')

gabf16_eda[(gabf16_eda.multiple_wins==1) & (gabf16_eda.brewery_type=='Regional')].nlargest(3,'dif_gabf16_norm')

brw_assoc = pd.read_csv('comp_breweries/brw_assoc_usa_gabf16.csv', sep=';', index_col=0)

print brw_assoc.shape
brw_assoc.head()

brewery_combos = [('CA', 'Brewpub'), ('CO', 'Brewpub'), ('OR', 'Micro'), ('WA', 'Regional'),
                  ('MT', 'Brewpub'), ('VA', 'Micro'), ('NC', 'Micro')
                  ]

np.random.seed(8)

for combo in brewery_combos:
    candidates = brw_assoc[(brw_assoc.state==combo[0]) & 
                           (brw_assoc.brewery_type==combo[1]) & 
                           (brw_assoc.gabf16_win==0)
                          ]['name']
    print combo
    print np.random.choice(candidates, 4, replace=False), '\n'

single_medals = gabf16_new[gabf16_new.medal_cnt==1][['brewery', 'name_Gtrends', 'state', 'brewery_type', 'medal', 'category']]
single_medals.head()

gold = [s for s in single_medals[single_medals.medal=='Gold']['name_Gtrends'].tolist() if s != u'n/a']
mean_idx_trends[gold].plot(legend=False, figsize=(16, 8))
plt.title('Gold Medal Brewery Search Hits (indexed)')
plt.ylabel('relative to own avg')

silver = [s for s in single_medals[single_medals.medal=='Silver']['name_Gtrends'].tolist() if s != u'n/a']
mean_idx_trends[silver].plot(legend=False, figsize=(16, 8))
plt.title('Silver Medal Brewery Search Hits (indexed)')
plt.ylabel('relative to own avg')

bronze = [s for s in single_medals[single_medals.medal=='Bronze']['name_Gtrends'].tolist() if s != u'n/a']
mean_idx_trends[bronze].plot(legend=False, figsize=(16, 8))
plt.title('Bronze Medal Brewery Search Hits (indexed)')
plt.ylabel('relative to own avg')

mean_idx_trends[gold].sum(axis=1)[pre_gabf16].describe(), mean_idx_trends[gold].sum(axis=1)[post_gabf16].describe()

mean_idx_trends[silver].sum(axis=1)[pre_gabf16].describe(), mean_idx_trends[silver].sum(axis=1)[post_gabf16].describe()

mean_idx_trends[bronze].sum(axis=1)[pre_gabf16].describe(), mean_idx_trends[bronze].sum(axis=1)[post_gabf16].describe()

single_medals_idx = pd.DataFrame({'gold': mean_idx_trends[gold].sum(axis=1),
                                  'silver': mean_idx_trends[silver].sum(axis=1),
                                  'bronze': mean_idx_trends[bronze].sum(axis=1),
                                  'gabf': 'GABF'
                                 })

single_medals_idx.loc[pre_gabf16, 'gabf']  = 'pre'
single_medals_idx.loc[post_gabf16, 'gabf']  = 'post'
single_medals_idx = single_medals_idx[['gold', 'silver', 'bronze', 'gabf']]

single_medals_idx.tail()

import seaborn as sns
sns.set_style("whitegrid")

df = single_medals_idx[single_medals_idx.gabf!='GABF'].set_index('gabf', append=True).stack().to_frame().reset_index()\
.rename(columns={'level_2': 'medal', 0: 'value'}) 

df.head()

fig, ax = plt.subplots()
fig.set_size_inches(11.7, 8.27)
sns.boxplot(x='medal', y='value', hue='gabf', data=df, palette="Set3")

sns.despine(trim=True)

pre_post_avgs.sort_values('dif_gabf16_norm', ascending=False)