Source code in Python

Get live tweets

This module establishes a connection with Twitter streaming API and downloads live tweets that are in English. The tweets are stored in 'tweets.json' .

import oauth2 as oauth
import urllib2 as urllib
import json

## Authorization
## Values not shown for privacy
api_key = "****"  
api_secret = "****"
access_token_key = "****"
access_token_secret = "****"
oauth_token = oauth.Token(key=access_token_key, secret=access_token_secret)
oauth_consumer = oauth.Consumer(key=api_key, secret=api_secret)
signature_method_hmac_sha1 = oauth.SignatureMethod_HMAC_SHA1()

## Send request to url
url = "https://stream.twitter.com/1.1/statuses/sample.json"
req = oauth.Request.from_consumer_and_token(oauth_consumer, token=oauth_token, 
                                            http_method="GET", http_url=url)
req.sign_request(signature_method_hmac_sha1, oauth_consumer, oauth_token)
url = req.to_url()

## Open the connection
opener = urllib.OpenerDirector()
http_handler = urllib.HTTPHandler(debuglevel=0)
https_handler = urllib.HTTPSHandler(debuglevel=0)
opener.add_handler(http_handler)
opener.add_handler(https_handler)

## Get the response
response = opener.open(url)

## Write response to file
f = open('tweets.json', 'w')
i = 0
for line in response:
  entry = json.loads(line)
  ## Keep English tweets only
  if u'lang' in entry and entry[u'lang'] == 'en' and u'text' in entry:
    json.dump(entry, f)
    f.write('\n')
    i += 1
  if i > 125000:
    break
f.close()

Sample output:

{"contributors": null, "truncated": false, "text": "@WelcomeToLevel7 - Now she just feels like she doesn't deserve it. And not in the humble way.* I think I'm going to take a break for...a -", "in_reply_to_status_id": 485787425491283969, "id": 485788441679257600, "favorite_count": 0, "source": "Twitter Web Client", "retweeted": false, "coordinates": null, "entities": {"user_mentions": [{"id": 1628911488, "indices": [0, 16], "id_str": "1628911488", "screen_name": "WelcomeToLevel7", "name": "Phil Coulson"}], "symbols": [], "trends": [], "hashtags": [], "urls": []}, "in_reply_to_screen_name": "WelcomeToLevel7", "id_str": "485788441679257600", "retweet_count": 0, "in_reply_to_user_id": 1628911488, "favorited": false, "user": {"follow_request_sent": null, "profile_use_background_image": true, "default_profile_image": false, "id": 1697653368, "verified": false, "profile_image_url_https": "https://pbs.twimg.com/profile_images/485787819541925890/5OUDWGiW_normal.jpeg", "profile_sidebar_fill_color": "D97E16", "profile_text_color": "B3A05B", "followers_count": 420, "profile_sidebar_border_color": "FF2A00", "id_str": "1697653368", "profile_background_color": "B8B8B8", "listed_count": 6, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/378800000058893099/3a32991a6b53a24365f66ac87b1ec7a2.jpeg", "utc_offset": -18000, "statuses_count": 13386, "description": "I don't remember much, but they tell me I was the star I was always meant to be. (Marvel RP.)", "friends_count": 194, "location": "Classified.", "profile_link_color": "ADBA82", "profile_image_url": "http://pbs.twimg.com/profile_images/485787819541925890/5OUDWGiW_normal.jpeg", "following": null, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/1697653368/1400431826", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/378800000058893099/3a32991a6b53a24365f66ac87b1ec7a2.jpeg", "name": "Carol Danvers", "lang": "en", "profile_background_tile": true, "favourites_count": 482, "screen_name": "SashesAreTrendy", "notifications": null, "url": "http://sashesaretrendy.tumblr.com", "created_at": "Sat Aug 24 23:44:42 +0000 2013", "contributors_enabled": false, "time_zone": "Central Time (US & Canada)", "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": "1628911488", "possibly_sensitive": false, "lang": "en", "created_at": "Sun Jul 06 14:12:44 +0000 2014", "filter_level": "medium", "in_reply_to_status_id_str": "485787425491283969", "place": null}
{"contributors": null, "truncated": false, "text": "RT @crochetyknits: Bamboo baby crochet converse style boots shoes booties MADE TO ORDER Please select size \u2026 http://t.co/KxGZRLeQll #etsymn\u2026", "in_reply_to_status_id": null, "id": 485788441662472192, "favorite_count": 0, "source": "BINGBINGF_blogger", "retweeted": false, "coordinates": null, "entities": {"user_mentions": [{"id": 1417482607, "indices": [3, 17], "id_str": "1417482607", "screen_name": "crochetyknits", "name": "crochetyknitsnbits "}], "symbols": [], "trends": [], "hashtags": [{"indices": [132, 140], "text": "etsymnt"}, {"indices": [139, 140], "text": "RedWhiteNavyBlue"}], "urls": [{"url": "http://t.co/KxGZRLeQll", "indices": [109, 131], "expanded_url": "http://etsy.me/SWKAxc", "display_url": "etsy.me/SWKAxc"}]}, "in_reply_to_screen_name": null, "id_str": "485788441662472192", "retweet_count": 0, "in_reply_to_user_id": null, "favorited": false, "retweeted_status": {"contributors": null, "truncated": false, "text": "Bamboo baby crochet converse style boots shoes booties MADE TO ORDER Please select size \u2026 http://t.co/KxGZRLeQll #etsymnt #RedWhiteNavyBlue", "in_reply_to_status_id": null, "id": 485756740583952384, "favorite_count": 0, "source": "etsy-fu", "retweeted": false, "coordinates": null, "entities": {"user_mentions": [], "symbols": [], "trends": [], "hashtags": [{"indices": [113, 121], "text": "etsymnt"}, {"indices": [122, 139], "text": "RedWhiteNavyBlue"}], "urls": [{"url": "http://t.co/KxGZRLeQll", "indices": [90, 112], "expanded_url": "http://etsy.me/SWKAxc", "display_url": "etsy.me/SWKAxc"}]}, "in_reply_to_screen_name": null, "id_str": "485756740583952384", "retweet_count": 6, "in_reply_to_user_id": null, "favorited": false, "user": {"follow_request_sent": null, "profile_use_background_image": true, "default_profile_image": false, "id": 1417482607, "verified": false, "profile_image_url_https": "https://pbs.twimg.com/profile_images/378800000141512667/c431922434375412dca3752b36164b18_normal.jpeg", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "followers_count": 8367, "profile_sidebar_border_color": "C0DEED", "id_str": "1417482607", "profile_background_color": "C0DEED", "listed_count": 86, "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "utc_offset": 3600, "statuses_count": 92553, "description": "Beautifully handmade & crochet baby clothes, shoes, dresses, hats & accessories are all individually made by me in my home workshop. Unique and one of a kind!", "friends_count": 9100, "location": "uk", "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/378800000141512667/c431922434375412dca3752b36164b18_normal.jpeg", "following": null, "geo_enabled": true, "profile_banner_url": "https://pbs.twimg.com/profile_banners/1417482607/1373963756", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "name": "crochetyknitsnbits ", "lang": "en", "profile_background_tile": false, "favourites_count": 351, "screen_name": "crochetyknits", "notifications": null, "url": "http://www.etsy.com/uk/shop/crochetyknitsnbits", "created_at": "Fri May 10 07:47:17 +0000 2013", "contributors_enabled": false, "time_zone": "London", "protected": false, "default_profile": true, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "lang": "en", "created_at": "Sun Jul 06 12:06:46 +0000 2014", "filter_level": "low", "in_reply_to_status_id_str": null, "place": null}, "user": {"follow_request_sent": null, "profile_use_background_image": true, "default_profile_image": false, "id": 2411648228, "verified": false, "profile_image_url_https": "https://pbs.twimg.com/profile_images/448604800389636096/IDnBnyEm_normal.png", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "followers_count": 3190, "profile_sidebar_border_color": "FFFFFF", "id_str": "2411648228", "profile_background_color": "C0DEED", "listed_count": 37, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/448610373537497088/DQ_3Frvz.png", "utc_offset": 0, "statuses_count": 124894, "description": null, "friends_count": 2768, "location": "", "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/448604800389636096/IDnBnyEm_normal.png", "following": null, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/2411648228/1398891291", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/448610373537497088/DQ_3Frvz.png", "name": "BINGBING Fashion", "lang": "en-gb", "profile_background_tile": false, "favourites_count": 55542, "screen_name": "BINGBINGFASHION", "notifications": null, "url": "http://www.thebingbing.com/fashion", "created_at": "Tue Mar 25 22:53:39 +0000 2014", "contributors_enabled": false, "time_zone": "Casablanca", "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "lang": "en", "created_at": "Sun Jul 06 14:12:44 +0000 2014", "filter_level": "medium", "in_reply_to_status_id_str": null, "place": null}
{"contributors": null, "truncated": false, "text": "@FA @bevington_a Totally agree with Jimmy Greaves in his Sunday People column with @tomhop_people", "in_reply_to_status_id": null, "id": 485788441641484288, "favorite_count": 0, "source": "Twitter for Android", "retweeted": false, "coordinates": null, "entities": {"user_mentions": [{"id": 39360648, "indices": [0, 3], "id_str": "39360648", "screen_name": "FA", "name": "The FA"}, {"id": 761803316, "indices": [4, 16], "id_str": "761803316", "screen_name": "bevington_a", "name": "ABevington"}, {"id": 112771580, "indices": [83, 97], "id_str": "112771580", "screen_name": "tomhop_people", "name": "Tom Hopkinson"}], "symbols": [], "trends": [], "hashtags": [], "urls": []}, "in_reply_to_screen_name": "FA", "id_str": "485788441641484288", "retweet_count": 0, "in_reply_to_user_id": 39360648, "favorited": false, "user": {"follow_request_sent": null, "profile_use_background_image": true, "default_profile_image": false, "id": 421100374, "verified": false, "profile_image_url_https": "https://pbs.twimg.com/profile_images/456267540876886017/3uMLB3nL_normal.jpeg", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "followers_count": 420, "profile_sidebar_border_color": "C0DEED", "id_str": "421100374", "profile_background_color": "F20C0C", "listed_count": 3, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/648701526/srs95mprzix2aspqlxbp.jpeg", "utc_offset": null, "statuses_count": 21433, "description": "Local nutter/Legend.  Love Stfc, love my Country.. Associate of Vinny McQuinny. #JFT96", "friends_count": 688, "location": "West Country.", "profile_link_color": "050101", "profile_image_url": "http://pbs.twimg.com/profile_images/456267540876886017/3uMLB3nL_normal.jpeg", "following": null, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/421100374/1403257587", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/648701526/srs95mprzix2aspqlxbp.jpeg", "name": "OfficialPaddyStavros", "lang": "en", "profile_background_tile": true, "favourites_count": 2429, "screen_name": "PaddyStavros", "notifications": null, "url": null, "created_at": "Fri Nov 25 14:10:03 +0000 2011", "contributors_enabled": false, "time_zone": null, "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": "39360648", "possibly_sensitive": false, "lang": "en", "created_at": "Sun Jul 06 14:12:44 +0000 2014", "filter_level": "medium", "in_reply_to_status_id_str": null, "place": null}
{"contributors": null, "truncated": false, "text": "RT @AmericanMadeMSL: 5 things American Makers @NikhilArora and @VelezAlejandro of @BTTRVentures learned starting a business. Read here -  h\u2026", "in_reply_to_status_id": null, "id": 485788441675071488, "favorite_count": 0, "source": "Twitter Web Client", "retweeted": false, "coordinates": null, "entities": {"user_mentions": [{"id": 1287365293, "indices": [3, 19], "id_str": "1287365293", "screen_name": "AmericanMadeMSL", "name": "American Made"}, {"id": 14929066, "indices": [46, 58], "id_str": "14929066", "screen_name": "NikhilArora", "name": "NikhilArora"}, {"id": 52832190, "indices": [63, 78], "id_str": "52832190", "screen_name": "VelezAlejandro", "name": "Alejandro Velez"}, {"id": 33819838, "indices": [82, 95], "id_str": "33819838", "screen_name": "BTTRVentures", "name": "Back to the Roots "}], "symbols": [], "trends": [], "hashtags": [], "urls": [{"url": "http://t.co/jHRcddx89Z", "indices": [139, 140], "expanded_url": "http://shout.lt/wpC1", "display_url": "shout.lt/wpC1"}]}, "in_reply_to_screen_name": null, "id_str": "485788441675071488", "retweet_count": 0, "in_reply_to_user_id": null, "favorited": false, "retweeted_status": {"contributors": null, "truncated": false, "text": "5 things American Makers @NikhilArora and @VelezAlejandro of @BTTRVentures learned starting a business. Read here -  http://t.co/jHRcddx89Z", "in_reply_to_status_id": null, "id": 482536634706329600, "favorite_count": 8, "source": "Shoutlet API", "retweeted": false, "coordinates": null, "entities": {"user_mentions": [{"id": 14929066, "indices": [25, 37], "id_str": "14929066", "screen_name": "NikhilArora", "name": "NikhilArora"}, {"id": 52832190, "indices": [42, 57], "id_str": "52832190", "screen_name": "VelezAlejandro", "name": "Alejandro Velez"}, {"id": 33819838, "indices": [61, 74], "id_str": "33819838", "screen_name": "BTTRVentures", "name": "Back to the Roots "}], "symbols": [], "trends": [], "hashtags": [], "urls": [{"url": "http://t.co/jHRcddx89Z", "indices": [117, 139], "expanded_url": "http://shout.lt/wpC1", "display_url": "shout.lt/wpC1"}]}, "in_reply_to_screen_name": null, "id_str": "482536634706329600", "retweet_count": 2, "in_reply_to_user_id": null, "favorited": false, "user": {"follow_request_sent": null, "profile_use_background_image": true, "default_profile_image": false, "id": 1287365293, "verified": false, "profile_image_url_https": "https://pbs.twimg.com/profile_images/465935295842508801/XbIV8g2e_normal.png", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "followers_count": 2498, "profile_sidebar_border_color": "FFFFFF", "id_str": "1287365293", "profile_background_color": "C0DEED", "listed_count": 43, "profile_background_image_url_https": "https://pbs.twimg.com/profile_background_images/840877123/c70e0dc9a6b8a77d95f02a0f08540b5f.jpeg", "utc_offset": null, "statuses_count": 696, "description": "Inspire. Make. Create.\r\nThe movement starts here.", "friends_count": 141, "location": "New York, NY", "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/465935295842508801/XbIV8g2e_normal.png", "following": null, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/1287365293/1403807862", "profile_background_image_url": "http://pbs.twimg.com/profile_background_images/840877123/c70e0dc9a6b8a77d95f02a0f08540b5f.jpeg", "name": "American Made", "lang": "en", "profile_background_tile": true, "favourites_count": 21, "screen_name": "AmericanMadeMSL", "notifications": null, "url": "http://americanmade.marthastewart.com", "created_at": "Fri Mar 22 00:46:56 +0000 2013", "contributors_enabled": false, "time_zone": null, "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "lang": "en", "created_at": "Fri Jun 27 14:51:13 +0000 2014", "filter_level": "low", "in_reply_to_status_id_str": null, "place": null}, "user": {"follow_request_sent": null, "profile_use_background_image": true, "default_profile_image": false, "id": 2598032916, "verified": false, "profile_image_url_https": "https://pbs.twimg.com/profile_images/483979470542348288/-YQ6T_h4_normal.jpeg", "profile_sidebar_fill_color": "DDEEF6", "profile_text_color": "333333", "followers_count": 21, "profile_sidebar_border_color": "C0DEED", "id_str": "2598032916", "profile_background_color": "C0DEED", "listed_count": 0, "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme1/bg.png", "utc_offset": null, "statuses_count": 29, "description": null, "friends_count": 122, "location": "", "profile_link_color": "0084B4", "profile_image_url": "http://pbs.twimg.com/profile_images/483979470542348288/-YQ6T_h4_normal.jpeg", "following": null, "geo_enabled": false, "profile_banner_url": "https://pbs.twimg.com/profile_banners/2598032916/1404224852", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme1/bg.png", "name": "Marj Weir", "lang": "en", "profile_background_tile": false, "favourites_count": 0, "screen_name": "PrepNServe", "notifications": null, "url": null, "created_at": "Tue Jul 01 14:06:59 +0000 2014", "contributors_enabled": false, "time_zone": null, "protected": false, "default_profile": true, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "lang": "en", "created_at": "Sun Jul 06 14:12:44 +0000 2014", "filter_level": "medium", "in_reply_to_status_id_str": null, "place": null}
{"contributors": null, "truncated": false, "text": "Come on #Federer . You've got this! #Wimbledon\u00a0", "in_reply_to_status_id": null, "id": 485788441653690368, "favorite_count": 0, "source": "Plume\u00a0for\u00a0Android", "retweeted": false, "coordinates": null, "entities": {"user_mentions": [], "symbols": [], "trends": [], "hashtags": [{"indices": [8, 16], "text": "Federer"}, {"indices": [36, 46], "text": "Wimbledon"}], "urls": []}, "in_reply_to_screen_name": null, "id_str": "485788441653690368", "retweet_count": 0, "in_reply_to_user_id": null, "favorited": false, "user": {"follow_request_sent": null, "profile_use_background_image": true, "default_profile_image": false, "id": 216406414, "verified": false, "profile_image_url_https": "https://pbs.twimg.com/profile_images/413187088842899456/4uTQOfaY_normal.jpeg", "profile_sidebar_fill_color": "EFEFEF", "profile_text_color": "333333", "followers_count": 149, "profile_sidebar_border_color": "EEEEEE", "id_str": "216406414", "profile_background_color": "131516", "listed_count": 1, "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme14/bg.gif", "utc_offset": 28800, "statuses_count": 15996, "description": "it's all a test, ace it", "friends_count": 381, "location": "Singapore", "profile_link_color": "009999", "profile_image_url": "http://pbs.twimg.com/profile_images/413187088842899456/4uTQOfaY_normal.jpeg", "following": null, "geo_enabled": false, "profile_background_image_url": "http://abs.twimg.com/images/themes/theme14/bg.gif", "name": "Syed Saleh", "lang": "en", "profile_background_tile": true, "favourites_count": 94, "screen_name": "syedsalehh", "notifications": null, "url": null, "created_at": "Tue Nov 16 16:53:28 +0000 2010", "contributors_enabled": false, "time_zone": "Singapore", "protected": false, "default_profile": false, "is_translator": false}, "geo": null, "in_reply_to_user_id_str": null, "possibly_sensitive": false, "lang": "en", "created_at": "Sun Jul 06 14:12:44 +0000 2014", "filter_level": "medium", "in_reply_to_status_id_str": null, "place": null}

Analyzing tweet sentiments

Get the tweet parameters

tweets = []
tweet_file = open('tweets.json')
for line in tweet_file:
  tweet = json.loads(line)
  text = coordinates = place = location = ''
  
  ## Get the tweet text
  text = tweet[u'text'].encode('utf-8')
  
  ## Get the tweet location
  if u'coordinates' in tweet and tweet[u'coordinates']:
    coordinates = tweet[u'coordinates'][u'coordinates']
    
  elif u'place' in tweet and tweet[u'place']:
    place = tweet[u'place'][u'full_name'].encode('utf-8')
    
  elif u'user' in tweet and tweet[u'user'][u'location']:
    location = tweet[u'user'][u'location'].encode('utf-8')
    
  tweets.append({'text':text, 'coordinates':coordinates, 
                 'place':place, 'location':location})

Evaluate the sentiment score

Get AFFIN scores from file 'AFINN-111.txt'. The scores range from +5 to -5:

def get_afinn_scores(file="AFINN-111.txt"): 
    afinn_file = open(file)
    scores = {}
    for line in afinn_file:
      term, score  = line.split("\t")
      scores[term] = int(score)
    return scores

Preview of AFFIN lexicon:

abandon   	-2
abandoned	-2
abandons	-2
abducted	-2
abduction	-2

Tokenize the tweet using Natural Language Tool Kit (NLTK) package. Find the corresponding AFFIN score to every word (if they exist in the lexicon), and calculate the overall sentiment of the tweet as the sum of the AFFIN scores:

import nltk

afinn_scores = get_afinn_scores()
tweets = get_tweets()

for tweet in tweets:
  ## Tokenize the tweet
  words = nltk.word_tokenize(tweet['text'])

  ## Get the total AFFIN score
  score = 0
  for w in words:
    if w.lower() in afinn_scores:
      score += afinn_scores[w.lower()]
      
  ## Store the value
  tweet['score'] = score

Get the tweet location

Load the States GeoJSON:

US_states = {'name': [], 'abbr': [], 'coord': []}
states_file = open("us-states.json")
features = json.load(states_file)[u'features']
for f in features:
  US_states['name'].append(f[u'properties'][u'name'].encode('utf-8'))
  US_states['abbr'].append(f[u'properties'][u'abbr'].encode('utf-8'))
  coord = f[u'geometry'][u'coordinates'][0]
  if len(coord)==1: coord = coord[0]
  US_states['coord'].append(coord)

Function for mapping coordinates to states:

def coord2state(coord):
    ## Check if the given location is within the state boundaries
    picked = []
    for i in range(len(US_states['name'])):
      ## Calculate the boundary box of the state
      xy = US_states['coord'][i]
      xmin = min(xy, key=lambda x:x[0])
      xmax = max(xy, key=lambda x:x[0])
      ymin = min(xy, key=lambda x:x[1])
      ymax = max(xy, key=lambda x:x[1])
      ## Check if the location is inside the box
      if (coord[0] >= xmin) and (coord[0] <= xmax) and 
        (coord[1] >= ymin) and (coord[1] <= ymax):
        picked.append(i)

    if len(picked) == 0:
      return ''

    if len(picked) == 1:
      return US_states['abbr'][picked[0]]

    ## If multiple states are found, pick the one that has
    ## the shortest distance from its center to the location
    d = []
    for k in picked:
      xcenter = 0.5 * sum(x for x,y in US_states['coord'][k]))
      ycenter = 0.5 * sum(y for x,y in US_states['coord'][k]))
      d.append( (x-xcenter)**2+(y-ycenter)**2 )
    idx = d.index(min(d))
    return US_states['abbr'][idx]

Extract state from tweet data:

def find_state(tweet):
    ## First look at the coordinates attribute
    if tweet['coordinates']:
      coord = tweet['coordinates']
      return coord2state(coord)
    ## Then look at the place attribute
    elif tweet[u'place']:
      place = tweet['place']
      place = " "+place+" "
      state_abbr = [s for s in US_states['abbr'] if " "+s+" " in place.upper()]
      if state_abbr:
        return state_abbr[0]
    ## Finally look at the user location attribute
    elif tweet[u'location']:
      location = tweet['location']
      location = " " + location + " "
      state_abbr = [s for s in US_states['abbr'] if " "+s+" " in location.upper()]
      state_name = [s for s in US_states['name'] if s.lower() in location.lower()]
      if state_abbr:
        return state_abbr[0]
      elif state_name:
        state_idx = US_states['name'].index(state_name[0])
        return US_states['abbr'][state_idx]
    return ''

Analyzing the mood of states

Associate the twitter sentiments with the states:

mood = {}
for tweet in tweets:
  state = find_state(tweet)
  if state: ## if the state information is available
    if state not in mood:
      mood[state] = [tweet['score']]
    else:
      mood[state].append(tweet['score'])

Save the data in a JSON file for visualization:

f = open('state_mood.json', 'w')
json.dump(mood, f)
f.close()