df = pd.read_csv('download/image-predictions.tsv', sep='\t')

df

#twitter-archive-enhanced.csv

df1 = pd.read_csv('twitter-archive-enhanced.csv')
df1

#tweepy

import tweepy

consumer_key = '-------------'
consumer_secret = '---------------'
access_token = '-----------------'
access_secret = '------------'

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)

auth.set_access_token(access_token, access_secret)

api = tweepy.API(auth, wait_on_rate_limit = True, wait_on_rate_limit_notify = True)

# public_tweets = api.home_timeline()
# for tweet in public_tweets:
#     print(tweet.text)

line_count = 0

with open('tweet_json.txt', encoding='utf-8', mode = 'w') as file:
    for i in range(len(df1)):
        try:
            result = api.get_status(str(df1.at[i,'tweet_id']))
            print(i, df1.at[i,'tweet_id'])
            file.write(str(result._json) + '\n')
            line_count += 1
        except tweepy.TweepError as e:
            print(i, e.reason)
            
    
print(line_count)

#Synthesize df2 from tweet_json.txt

# I assigned the line_count value from previous cell here directly in 
# order to run this cell individually since it will take some time to
# query twitter API

line_count = 2345

df2_list =[]

regex_id = '\'id\': ([0-9]+)'
regex_retweet = '\'retweet_count\': ([0-9]+)'
regex_fav = '\'favorite_count\': ([0-9]+)'

r_id = re.compile(regex_id)
r_ret = re.compile(regex_retweet)
r_fav = re.compile(regex_fav)

count = 0
with open('tweet_json.txt', encoding='utf-8') as file:
    while count < line_count:
        title = file.readline()
#         line1 = title.split(',')
#         id = line1[1][7:len(line1[1])]
        idre = r_id.findall(title)
        favc = r_fav.findall(title)
        retc = r_ret.findall(title)
        
#         fav_count = line1[len(line1)-6][19:len(line1[len(line1)-6])]
#         ret_count = line1[len(line1)-7][18:len(line1[len(line1)-7])]

        df2_list.append({'tweet_id' : str(idre[0]),
                        'retweet_count' : retc[0],
                        'favorite_count' : favc[0]})
        #print(count, idre[0], favc[0], retc[0])
        count += 1
        
df2 = pd.DataFrame(df2_list, columns = ['tweet_id', 'retweet_count', 'favorite_count'])

df2

df[df.tweet_id.duplicated()]

df1[df1.tweet_id.duplicated()]

df2[df2.id.duplicated()]

sum(df.tweet_id.isnull())

sum(df1.tweet_id.isnull())

sum(df1.rating_numerator.isnull())

sum(df2.id.isnull())

df1.rating_denominator.value_counts()

df1.rating_numerator.value_counts()

df1[(df1['rating_numerator'] >= 20)]

df1[(df1['rating_denominator'] < 10)]

pd.options.display.max_colwidth = -1

df1[(df1['name'].str.islower() == True)]

df1[df1.name.duplicated()]

df1[df1.name.isnull()]

df1[(df1['text'].str.slice(start=0, stop=2) == 'RT')]

df1[df1.retweeted_status_id.notnull()]

df1_clean['name'].describe()

pd.options.display.max_colwidth = -1


df1[(df1['source'].str.slice(start=72, stop=-4) != 'iPhone')]


# df1['source'].str.slice(start=72, stop=-4)

# df1['source'].str.slice(start=0, stop=-4)

df2['tweet_id'].describe()

df1_clean[(df1_clean['text'].str.slice(start=0, stop=2) == 'RT')]

# df1_clean

df1_clean.source.value_counts()

df_clean[['p1', 'p2', 'p3']]

regex_name2 = 'named ([A-Z][a-z]+)|is ([A-Z][a-z]+)'

r_name2 = re.compile(regex_name2)
                   
                         
for i in range(len(df1_clean)):
    if df1_clean.loc[i, 'name'].islower():
                         
        df1_text = df1_clean.loc[i, 'text']
        df1_name = r_name2.findall(df1_text)
        
        if len(df1_name) == 1:
            
            if df1_name[0][0]:
                df1_clean.at[i, 'name'] = df1_name[0][0]
                print(df1_name)
                
            if df1_name[0][1]:
                df1_clean.at[i, 'name'] = df1_name[0][1]
                print(df1_name)
   
        if len(df1_name) == 0:
            df1_clean.at[i, 'name'] = "None"
            

df1_clean[(df1_clean['tweet_id'] == 666701168228331520)]

df1_clean[(df1_clean['tweet_id'] == 748692773788876800)]

df1_clean.sample(200)

df1_clean.sample(200)

pd.options.display.max_colwidth = -1
df1_clean[['rating_numerator', 'rating_denominator', 'text' ]][(df1_clean['rating_denominator'] != 10)]

df1_clean.rating_denominator.value_counts()

df1_clean

df2_clean['tweet_id'].describe()

df_master_clean = pd.merge(df_clean, df1_clean, on=['tweet_id'])

df_master_clean = pd.merge(df_master_clean, df2_clean, on=['tweet_id'])

df_master_clean

#The breeds of dog in the top 20 favorited tweets.

df_top_fav = df_master.sort_values('favorite_count', ascending=False).head(20)[['p1', 'p1_dog', 'favorite_count']]

df_top_fav

#The breeds of dog in the top 20 retweeted tweets.

df_top_ret = df_master.sort_values('retweet_count', ascending=False).head(20)[['p1', 'p1_dog', 'retweet_count']]

df_top_ret

df_top_rated = df_master[['p1', 'p1_dog', 'rating']][(df_master['rating'] >= 1.4)].sort_values('rating', ascending=False)

df_top_rated

df_top_dogs = df_top_fav.append(df_top_ret).append(df_top_rated)

df_top_dogs = df_top_dogs[(df_top_dogs['p1_dog'] == True)]


df_top_dogs

df_top_dogs.p1.value_counts().sort_values(ascending=True).plot(kind='barh', figsize=(8, 8), rot=0)
# plt.axis('off')
# plt.axis('equal')
plt.title('The most popular breed found')

#This part is not in the report

df_master.source.value_counts()

df_master.source.value_counts().plot(kind='pie', figsize=(8, 8), autopct='%.2f')
plt.axis('off')
plt.axis('equal')
plt.title('Source pie chart')