diff --git a/README.md b/README.md index d5194ad..f2af271 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,7 @@ -**NOTE:** If you're trying to run the RollDieDynamic.py script in Chapter 1 and the window does not appear, this is due to a known issue in the software for which a fix is already in process. You can instead run the program with - ->`ipython -i RollDieDynamic.py 6000 1` - -Then terminate IPython interactive mode (Ctrl + D twice) after you close the script's window. - -# PythonForProgrammers +# Python for Programmers This repository contains the source code and supporting files associated with our book Python for Programmers, which is a subset of our book Intro to Python for Computer Science and Data Science: Learning to Program with AI, Big Data and the Cloud. -![Cover image for Python for Programmers](http://deitel.com/bookresources/PythonFP/PythonForProgrammersCover.png) +![Cover image for Python for Programmers](https://deitel.com/wp-content/uploads/2020/01/python-for-programmers.jpg) These files are Copyright 2019 by Pearson Education, Inc. All Rights Reserved. diff --git a/examples/ch01/RollDieDynamic.py b/examples/ch01/RollDieDynamic.py index ebb6c48..433d08a 100755 --- a/examples/ch01/RollDieDynamic.py +++ b/examples/ch01/RollDieDynamic.py @@ -14,7 +14,7 @@ def update(frame_number, rolls, faces, frequencies): # reconfigure plot for updated die frequencies plt.cla() # clear old contents contents of current Figure - axes = sns.barplot(faces, frequencies, palette='bright') # new bars + axes = sns.barplot(x=faces, y=frequencies, palette='bright') # new bars axes.set_title(f'Die Frequencies for {sum(frequencies):,} Rolls') axes.set(xlabel='Die Value', ylabel='Frequency') axes.set_ylim(top=max(frequencies) * 1.10) # scale y-axis by 10% diff --git a/examples/ch05/snippets_ipynb/05_09.ipynb b/examples/ch05/snippets_ipynb/05_09.ipynb index 289fe9e..87576fd 100755 --- a/examples/ch05/snippets_ipynb/05_09.ipynb +++ b/examples/ch05/snippets_ipynb/05_09.ipynb @@ -148,7 +148,7 @@ "outputs": [], "source": [ "if key in numbers:\n", - " print(f'found {key} at index {numbers.index(search_key)}')\n", + " print(f'found {key} at index {numbers.index(key)}')\n", "else:\n", " print(f'{key} not found')" ] @@ -204,4 +204,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/examples/ch05/snippets_py/05_09.py b/examples/ch05/snippets_py/05_09.py index 7f7765e..60739bc 100755 --- a/examples/ch05/snippets_py/05_09.py +++ b/examples/ch05/snippets_py/05_09.py @@ -29,7 +29,7 @@ key = 1000 if key in numbers: - print(f'found {key} at index {numbers.index(search_key)}') + print(f'found {key} at index {numbers.index(key)}') else: print(f'{key} not found') diff --git a/examples/ch06/RollDieDynamic.py b/examples/ch06/RollDieDynamic.py index c7bf9ae..9257e69 100755 --- a/examples/ch06/RollDieDynamic.py +++ b/examples/ch06/RollDieDynamic.py @@ -14,7 +14,7 @@ def update(frame_number, rolls, faces, frequencies): # reconfigure plot for updated die frequencies plt.cla() # clear old contents contents of current Figure - axes = sns.barplot(faces, frequencies, palette='bright') # new bars + axes = sns.barplot(x=faces, y=frequencies, palette='bright') # new bars axes.set_title(f'Die Frequencies for {sum(frequencies):,} Rolls') axes.set(xlabel='Die Value', ylabel='Frequency') axes.set_ylim(top=max(frequencies) * 1.10) # scale y-axis by 10% diff --git a/examples/ch12/_READ_ME_FIRST b/examples/ch12/_READ_ME_FIRST new file mode 100644 index 0000000..e59bc95 --- /dev/null +++ b/examples/ch12/_READ_ME_FIRST @@ -0,0 +1,19 @@ +**Upgrading from Twitter v1.1 to Twitter v2** +On August 18, 2022, we discovered that new Twitter developer accounts cannot access the Twitter version 1.1 APIs on which we based Chapter 12, Data Mining Twitter, and two case studies in Chapter 16, Big Data: Hadoop, Spark, NoSQL and IoT. + +Twitter users who already had Twitter developer accounts can still access the Twitter version 1.1 APIs, but viewers will not fall into this category. + +We’re updating Chapter 12 and the two case studies in Chapter 16 to the Twitter version 2 APIs. We anticipate this could take two to four weeks as we: +1. master the new Twitter v2 APIs, +2. update our Chapter 12 and 16 code examples, and +3. update the source-code files and Jupyter Notebooks in this repository. + +Once completed, we’ll: +• post an updated version of Chapter 12 and the relevant sections of Chapter 16 to the book’s webpages at https://informit.com and https://deitel.com, +• post updated files to this GitHub repository. + +If you have any questions, please email paul@deitel.com. + + + + diff --git a/examples/ch12_TwitterV1.1/_READ_ME_FIRST b/examples/ch12_TwitterV1.1/_READ_ME_FIRST new file mode 100644 index 0000000..ce7b389 --- /dev/null +++ b/examples/ch12_TwitterV1.1/_READ_ME_FIRST @@ -0,0 +1,14 @@ +# Upgrading from Twitter v1.1 to Twitter v2 + +On August 18, 2022, we discovered that **new** Twitter developer accounts cannot access the Twitter version 1.1 APIs on which we based Chapter 12, Data Mining Twitter, and two case studies in Chapter 16, Big Data: Hadoop, Spark, NoSQL and IoT. + +Twitter users who already had Twitter developer accounts can still access the Twitter version 1.1 APIs, but most students and some instructors will not fall into this category. + +We’ve already rewritten Chapter 12 to the Twitter version 2 APIs and will be rewriting the two Twitter-based Chapter 16 case studies soon. + +The updated chapter is posted on the book's webpage: +https://deitel.com/python-for-programmers-book/ + +Once completed, we’ll post updated versions of the Chapter 16 Twitter-based case studies. + +Questions? Please email paul@deitel.com. diff --git a/examples/ch12/keys.py b/examples/ch12_TwitterV1.1/keys.py similarity index 100% rename from examples/ch12/keys.py rename to examples/ch12_TwitterV1.1/keys.py diff --git a/examples/ch12/locationlistener.py b/examples/ch12_TwitterV1.1/locationlistener.py similarity index 100% rename from examples/ch12/locationlistener.py rename to examples/ch12_TwitterV1.1/locationlistener.py diff --git a/examples/ch12/sentimentlistener.py b/examples/ch12_TwitterV1.1/sentimentlistener.py similarity index 100% rename from examples/ch12/sentimentlistener.py rename to examples/ch12_TwitterV1.1/sentimentlistener.py diff --git a/examples/ch12_TwitterV1.1/snippets_ipynb/12_07-11.ipynb b/examples/ch12_TwitterV1.1/snippets_ipynb/12_07-11.ipynb new file mode 100755 index 0000000..30f18a6 --- /dev/null +++ b/examples/ch12_TwitterV1.1/snippets_ipynb/12_07-11.ipynb @@ -0,0 +1,781 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**_Note: This notebook contains ALL the code for Sections 12.7 through 12.11 because all the snippets in these sections are consecutively numbered in the text._**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.7 Authenticating with Twitter Via Tweepy to Access Twitter v2 APIs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tweepy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import keys" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a Client Object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = tweepy.Client(bearer_token=keys.bearer_token,\n", + " wait_on_rate_limit=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.8 Getting Information About a Twitter Account" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "nasa = client.get_user(username='NASA', \n", + " user_fields=['description', 'public_metrics'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### tweepy.Response Object" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting a User’s Basic Account Information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.username" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "nasa.data.description" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Getting the Number of Accounts That Follow This Account and the Number of Accounts This Account Follows" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.public_metrics['followers_count']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.public_metrics['following_count']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting Your Own Account’s Information" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.9 Intro to Tweepy `Paginator`s: Getting More than One Page of Results" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# 12.9.1 Determining an Account’s Followers " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "followers = []" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a `Paginator`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "paginator = tweepy.Paginator(\n", + " client.get_users_followers, nasa.data.id, max_results=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting Results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for follower in paginator.flatten(limit=10):\n", + " followers.append(follower.username)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('Followers:', \n", + " ' '.join(sorted(followers, key=lambda s: s.lower())))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.9.2 Determining Whom an Account Follows " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "following = []" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "paginator = tweepy.Paginator(\n", + " client.get_users_following, nasa.data.id, max_results=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for user_followed in paginator.flatten(limit=10):\n", + " following.append(user_followed.username)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('Following:', \n", + " ' '.join(sorted(following, key=lambda s: s.lower())))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.9.3 Getting a User’s Recent Tweets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa_tweets = client.get_users_tweets(\n", + " id=nasa.data.id, max_results=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for tweet in nasa_tweets.data:\n", + " print(f\"NASA: {tweet.data['text']}\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### Grabbing Recent Tweets from Your Own Timeline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.10 Searching Recent Tweets; Intro to Twitter v2 API Search Operators" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Utility Function `print_tweets` from `tweetutilities.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tweetutilities import print_tweets" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "def print_tweets(tweets):\n", + " # translator to autodetect source language and return English\n", + " translator = GoogleTranslator(source='auto', target='en')\n", + "\n", + " \"\"\"For each tweet in tweets, display the username of the sender\n", + " and tweet text. If the language is not English, translate the text \n", + " with the deep-translator library's GoogleTranslator.\"\"\"\n", + " for tweet, user in zip(tweets.data, tweets.includes['users']):\n", + " print(f'{user.username}:', end=' ')\n", + "\n", + " if 'en' in tweet.lang:\n", + " print(f'{tweet.text}\\n')\n", + " elif 'und' not in tweet.lang: # translate to English first\n", + " print(f'\\n ORIGINAL: {tweet.text}')\n", + " print(f'TRANSLATED: {translator.translate(tweet.text)}\\n')\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching for Specific Words" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweets = client.search_recent_tweets(\n", + " query='Webb Space Telescope', \n", + " expansions=['author_id'], tweet_fields=['lang'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_tweets(tweets)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching with Twitter v2 API Search Operators" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching with Twitter v2 API Search Operators" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Operator Documentation and Tutorial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching for Tweets From NASA Containing Links" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweets = client.search_recent_tweets(\n", + " query='from:NASA has:links', \n", + " expansions=['author_id'], tweet_fields=['lang'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_tweets(tweets)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching for a Hashtag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweets = client.search_recent_tweets(query='#metaverse', \n", + " expansions=['author_id'], tweet_fields=['lang'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_tweets(tweets)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.11 Spotting Trending Topics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "auth = tweepy.OAuth2BearerHandler(keys.bearer_token)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "api = tweepy.API(auth=auth, wait_on_rate_limit=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.11.1 Places with Trending Topics\n", + "**Note: This part of the Twitter APIs has not been migrated from v1.1 to v2 yet and is accessible only to \"Elevated\" and \"Academic Research\" access.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "available_trends = api.available_trends()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(available_trends)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "available_trends[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "available_trends[1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.11.2 Getting a List of Trending Topics\n", + "### Worldwide Trending Topics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "world_trends = api.get_place_trends(id=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trends_list = world_trends[0]['trends']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trends_list[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trends_list = [t for t in trends_list if t['tweet_volume']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from operator import itemgetter " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trends_list.sort(key=itemgetter('tweet_volume'), reverse=True) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for trend in trends_list:\n", + " print(trend['name'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### New York City Trending Topics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "nyc_trends = api.get_place_trends(id=2459115) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nyc_list = nyc_trends[0]['trends']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nyc_list = [t for t in nyc_list if t['tweet_volume']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nyc_list.sort(key=itemgetter('tweet_volume'), reverse=True) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for trend in nyc_list[:5]:\n", + " print(trend['name'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.11.3 Create a Word Cloud from Trending Topics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "topics = {}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "for trend in nyc_list:\n", + " topics[trend['name']] = trend['tweet_volume']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from wordcloud import WordCloud" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wordcloud = WordCloud(width=1600, height=900,\n", + " prefer_horizontal=0.5, min_font_size=10, colormap='prism', \n", + " background_color='white') " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wordcloud = wordcloud.fit_words(topics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wordcloud = wordcloud.to_file('TrendingTwitter.png')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**NOTE: The following code displays the image in a Jupyter Notebook**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import Image\n", + "Image(filename='TrendingTwitter.png', width=400)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "##########################################################################\n", + "# (C) Copyright 2022 by Deitel & Associates, Inc. and #\n", + "# Pearson Education, Inc. All Rights Reserved. #\n", + "# #\n", + "# DISCLAIMER: The authors and publisher of this book have used their #\n", + "# best efforts in preparing the book. These efforts include the #\n", + "# development, research, and testing of the theories and programs #\n", + "# to determine their effectiveness. The authors and publisher make #\n", + "# no warranty of any kind, expressed or implied, with regard to these #\n", + "# programs or to the documentation contained in these books. The authors #\n", + "# and publisher shall not be liable in any event for incidental or #\n", + "# consequential damages in connection with, or arising out of, the #\n", + "# furnishing, performance, or use of these programs. #\n", + "##########################################################################\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/ch12/snippets_ipynb/12_12.ipynb b/examples/ch12_TwitterV1.1/snippets_ipynb/12_12.ipynb similarity index 100% rename from examples/ch12/snippets_ipynb/12_12.ipynb rename to examples/ch12_TwitterV1.1/snippets_ipynb/12_12.ipynb diff --git a/examples/ch12/snippets_ipynb/12_13_02.ipynb b/examples/ch12_TwitterV1.1/snippets_ipynb/12_13_02.ipynb similarity index 100% rename from examples/ch12/snippets_ipynb/12_13_02.ipynb rename to examples/ch12_TwitterV1.1/snippets_ipynb/12_13_02.ipynb diff --git a/examples/ch12/snippets_ipynb/12_14.ipynb b/examples/ch12_TwitterV1.1/snippets_ipynb/12_14.ipynb similarity index 100% rename from examples/ch12/snippets_ipynb/12_14.ipynb rename to examples/ch12_TwitterV1.1/snippets_ipynb/12_14.ipynb diff --git a/examples/ch12/snippets_ipynb/12_15.ipynb b/examples/ch12_TwitterV1.1/snippets_ipynb/12_15.ipynb similarity index 100% rename from examples/ch12/snippets_ipynb/12_15.ipynb rename to examples/ch12_TwitterV1.1/snippets_ipynb/12_15.ipynb diff --git a/examples/ch12/snippets_ipynb/README.txt b/examples/ch12_TwitterV1.1/snippets_ipynb/README.txt similarity index 100% rename from examples/ch12/snippets_ipynb/README.txt rename to examples/ch12_TwitterV1.1/snippets_ipynb/README.txt diff --git a/examples/ch12/snippets_ipynb/files/art/check.png b/examples/ch12_TwitterV1.1/snippets_ipynb/files/art/check.png similarity index 100% rename from examples/ch12/snippets_ipynb/files/art/check.png rename to examples/ch12_TwitterV1.1/snippets_ipynb/files/art/check.png diff --git a/examples/ch12/snippets_ipynb/keys.py b/examples/ch12_TwitterV1.1/snippets_ipynb/keys.py similarity index 100% rename from examples/ch12/snippets_ipynb/keys.py rename to examples/ch12_TwitterV1.1/snippets_ipynb/keys.py diff --git a/examples/ch12/snippets_ipynb/locationlistener.py b/examples/ch12_TwitterV1.1/snippets_ipynb/locationlistener.py similarity index 100% rename from examples/ch12/snippets_ipynb/locationlistener.py rename to examples/ch12_TwitterV1.1/snippets_ipynb/locationlistener.py diff --git a/examples/ch12/snippets_ipynb/sentimentlistener.py b/examples/ch12_TwitterV1.1/snippets_ipynb/sentimentlistener.py similarity index 100% rename from examples/ch12/snippets_ipynb/sentimentlistener.py rename to examples/ch12_TwitterV1.1/snippets_ipynb/sentimentlistener.py diff --git a/examples/ch12/snippets_ipynb/tweetlistener.py b/examples/ch12_TwitterV1.1/snippets_ipynb/tweetlistener.py similarity index 100% rename from examples/ch12/snippets_ipynb/tweetlistener.py rename to examples/ch12_TwitterV1.1/snippets_ipynb/tweetlistener.py diff --git a/examples/ch12/snippets_ipynb/tweetutilities.py b/examples/ch12_TwitterV1.1/snippets_ipynb/tweetutilities.py similarity index 100% rename from examples/ch12/snippets_ipynb/tweetutilities.py rename to examples/ch12_TwitterV1.1/snippets_ipynb/tweetutilities.py diff --git a/examples/ch12/snippets_py/12_07-11.py b/examples/ch12_TwitterV1.1/snippets_py/12_07-11.py similarity index 100% rename from examples/ch12/snippets_py/12_07-11.py rename to examples/ch12_TwitterV1.1/snippets_py/12_07-11.py diff --git a/examples/ch12/snippets_py/12_12.py b/examples/ch12_TwitterV1.1/snippets_py/12_12.py similarity index 100% rename from examples/ch12/snippets_py/12_12.py rename to examples/ch12_TwitterV1.1/snippets_py/12_12.py diff --git a/examples/ch12/snippets_py/12_13_02.py b/examples/ch12_TwitterV1.1/snippets_py/12_13_02.py similarity index 100% rename from examples/ch12/snippets_py/12_13_02.py rename to examples/ch12_TwitterV1.1/snippets_py/12_13_02.py diff --git a/examples/ch12/snippets_py/12_15_01.py b/examples/ch12_TwitterV1.1/snippets_py/12_15_01.py similarity index 100% rename from examples/ch12/snippets_py/12_15_01.py rename to examples/ch12_TwitterV1.1/snippets_py/12_15_01.py diff --git a/examples/ch12/snippets_py/README.txt b/examples/ch12_TwitterV1.1/snippets_py/README.txt similarity index 100% rename from examples/ch12/snippets_py/README.txt rename to examples/ch12_TwitterV1.1/snippets_py/README.txt diff --git a/examples/ch12/snippets_py/keys.py b/examples/ch12_TwitterV1.1/snippets_py/keys.py similarity index 100% rename from examples/ch12/snippets_py/keys.py rename to examples/ch12_TwitterV1.1/snippets_py/keys.py diff --git a/examples/ch12/snippets_py/locationlistener.py b/examples/ch12_TwitterV1.1/snippets_py/locationlistener.py similarity index 100% rename from examples/ch12/snippets_py/locationlistener.py rename to examples/ch12_TwitterV1.1/snippets_py/locationlistener.py diff --git a/examples/ch12/snippets_py/sentimentlistener.py b/examples/ch12_TwitterV1.1/snippets_py/sentimentlistener.py similarity index 100% rename from examples/ch12/snippets_py/sentimentlistener.py rename to examples/ch12_TwitterV1.1/snippets_py/sentimentlistener.py diff --git a/examples/ch12/snippets_py/tweetlistener.py b/examples/ch12_TwitterV1.1/snippets_py/tweetlistener.py similarity index 100% rename from examples/ch12/snippets_py/tweetlistener.py rename to examples/ch12_TwitterV1.1/snippets_py/tweetlistener.py diff --git a/examples/ch12/snippets_py/tweetutilities.py b/examples/ch12_TwitterV1.1/snippets_py/tweetutilities.py similarity index 100% rename from examples/ch12/snippets_py/tweetutilities.py rename to examples/ch12_TwitterV1.1/snippets_py/tweetutilities.py diff --git a/examples/ch12/tweetlistener.py b/examples/ch12_TwitterV1.1/tweetlistener.py similarity index 100% rename from examples/ch12/tweetlistener.py rename to examples/ch12_TwitterV1.1/tweetlistener.py diff --git a/examples/ch12/tweetutilities.py b/examples/ch12_TwitterV1.1/tweetutilities.py similarity index 100% rename from examples/ch12/tweetutilities.py rename to examples/ch12_TwitterV1.1/tweetutilities.py diff --git a/examples/ch12_TwitterV2/_READ_ME_FIRST b/examples/ch12_TwitterV2/_READ_ME_FIRST new file mode 100644 index 0000000..ce7b389 --- /dev/null +++ b/examples/ch12_TwitterV2/_READ_ME_FIRST @@ -0,0 +1,14 @@ +# Upgrading from Twitter v1.1 to Twitter v2 + +On August 18, 2022, we discovered that **new** Twitter developer accounts cannot access the Twitter version 1.1 APIs on which we based Chapter 12, Data Mining Twitter, and two case studies in Chapter 16, Big Data: Hadoop, Spark, NoSQL and IoT. + +Twitter users who already had Twitter developer accounts can still access the Twitter version 1.1 APIs, but most students and some instructors will not fall into this category. + +We’ve already rewritten Chapter 12 to the Twitter version 2 APIs and will be rewriting the two Twitter-based Chapter 16 case studies soon. + +The updated chapter is posted on the book's webpage: +https://deitel.com/python-for-programmers-book/ + +Once completed, we’ll post updated versions of the Chapter 16 Twitter-based case studies. + +Questions? Please email paul@deitel.com. diff --git a/examples/ch12_TwitterV2/keys.py b/examples/ch12_TwitterV2/keys.py new file mode 100755 index 0000000..bf7c356 --- /dev/null +++ b/examples/ch12_TwitterV2/keys.py @@ -0,0 +1,2 @@ +bearer_token = 'YourBearerToken' +mapquest_key = 'YourAPIKey' \ No newline at end of file diff --git a/examples/ch12_TwitterV2/locationlistener.py b/examples/ch12_TwitterV2/locationlistener.py new file mode 100755 index 0000000..f154d61 --- /dev/null +++ b/examples/ch12_TwitterV2/locationlistener.py @@ -0,0 +1,59 @@ +# locationlistener.py +"""Receives tweets matching a search string and stores a list of +dictionaries containing each tweet's username/text/location.""" +import tweepy +from tweetutilities import get_tweet_content + +class LocationListener(tweepy.StreamingClient): + """Handles incoming Tweet stream to get location data.""" + + def __init__(self, bearer_token, counts_dict, + tweets_list, topic, limit=10): + """Configure the LocationListener.""" + self.tweets_list = tweets_list + self.counts_dict = counts_dict + self.topic = topic + self.TWEET_LIMIT = limit + super().__init__(bearer_token, wait_on_rate_limit=True) + + def on_response(self, response): + """Called when Twitter pushes a new tweet to you.""" + + # get tweet's username, text and location + tweet_data = get_tweet_content(response) + + # ignore retweets and tweets that do not contain the topic + if (tweet_data['text'].startswith('RT') or + self.topic.lower() not in tweet_data['text'].lower()): + return + + self.counts_dict['total_tweets'] += 1 # it's an original tweet + + # ignore tweets with no location + if not tweet_data.get('location'): + return + + self.counts_dict['locations'] += 1 # user account has location + self.tweets_list.append(tweet_data) # store the tweet + print(f"{tweet_data['username']}: {tweet_data['text']}\n") + + # if TWEET_LIMIT is reached, terminate streaming + if self.counts_dict['locations'] == self.TWEET_LIMIT: + self.disconnect() + + + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/sentimentlistener.py b/examples/ch12_TwitterV2/sentimentlistener.py new file mode 100755 index 0000000..2cc45ec --- /dev/null +++ b/examples/ch12_TwitterV2/sentimentlistener.py @@ -0,0 +1,106 @@ +# sentimentlisener.py +"""Searches for tweets that match a search string and tallies +the number of positive, neutral and negative tweets.""" +import keys +import preprocessor as p +import sys +from textblob import TextBlob +import tweepy + +class SentimentListener(tweepy.StreamingClient): + """Handles incoming Tweet stream.""" + + def __init__(self, bearer_token, sentiment_dict, topic, limit=10): + """Configure the SentimentListener.""" + self.sentiment_dict = sentiment_dict + self.tweet_count = 0 + self.topic = topic + self.TWEET_LIMIT = limit + + # set tweet-preprocessor to remove URLs/reserved words + p.set_options(p.OPT.URL, p.OPT.RESERVED) + super().__init__(bearer_token, wait_on_rate_limit=True) + + def on_response(self, response): + """Called when Twitter pushes a new tweet to you.""" + + # if the tweet is not a retweet + if not response.data.text.startswith('RT'): + text = p.clean(response.data.text) # clean the tweet + + # ignore tweet if the topic is not in the tweet text + if self.topic.lower() not in text.lower(): + return + + # update self.sentiment_dict with the polarity + blob = TextBlob(text) + if blob.sentiment.polarity > 0: + sentiment = '+' + self.sentiment_dict['positive'] += 1 + elif blob.sentiment.polarity == 0: + sentiment = ' ' + self.sentiment_dict['neutral'] += 1 + else: + sentiment = '-' + self.sentiment_dict['negative'] += 1 + + # display the tweet + username = response.includes['users'][0].username + print(f'{sentiment} {username}: {text}\n') + + self.tweet_count += 1 # track number of tweets processed + + # if TWEET_LIMIT is reached, terminate streaming + if self.tweet_count == self.TWEET_LIMIT: + self.disconnect() + +def main(): + # get search term and number of tweets + search_key = sys.argv[1] + limit = int(sys.argv[2]) # number of tweets to tally + + # set up the sentiment dictionary + sentiment_dict = {'positive': 0, 'neutral': 0, 'negative': 0} + + # create the StreamingClient subclass object + sentiment_listener = SentimentListener(keys.bearer_token, + sentiment_dict, search_key, limit) + + # redirect sys.stderr to sys.stdout + sys.stderr = sys.stdout + + # delete existing stream rules + rules = sentiment_listener.get_rules().data + rule_ids = [rule.id for rule in rules] + sentiment_listener.delete_rules(rule_ids) + + # create stream rule + sentiment_listener.add_rules( + tweepy.StreamRule(f'{search_key} lang:en')) + + # start filtering English tweets containing search_key + sentiment_listener.filter(expansions=['author_id']) + + print(f'Tweet sentiment for "{search_key}"') + print('Positive:', sentiment_dict['positive']) + print(' Neutral:', sentiment_dict['neutral']) + print('Negative:', sentiment_dict['negative']) + +# call main if this file is executed as a script +if __name__ == '__main__': + main() + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_ipynb/12_07-11.ipynb b/examples/ch12_TwitterV2/snippets_ipynb/12_07-11.ipynb new file mode 100755 index 0000000..30f18a6 --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/12_07-11.ipynb @@ -0,0 +1,781 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**_Note: This notebook contains ALL the code for Sections 12.7 through 12.11 because all the snippets in these sections are consecutively numbered in the text._**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.7 Authenticating with Twitter Via Tweepy to Access Twitter v2 APIs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tweepy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import keys" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a Client Object" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = tweepy.Client(bearer_token=keys.bearer_token,\n", + " wait_on_rate_limit=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.8 Getting Information About a Twitter Account" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "nasa = client.get_user(username='NASA', \n", + " user_fields=['description', 'public_metrics'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### tweepy.Response Object" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting a User’s Basic Account Information" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.username" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "nasa.data.description" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Getting the Number of Accounts That Follow This Account and the Number of Accounts This Account Follows" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.public_metrics['followers_count']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa.data.public_metrics['following_count']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting Your Own Account’s Information" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.9 Intro to Tweepy `Paginator`s: Getting More than One Page of Results" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# 12.9.1 Determining an Account’s Followers " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "followers = []" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a `Paginator`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "paginator = tweepy.Paginator(\n", + " client.get_users_followers, nasa.data.id, max_results=5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Getting Results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for follower in paginator.flatten(limit=10):\n", + " followers.append(follower.username)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('Followers:', \n", + " ' '.join(sorted(followers, key=lambda s: s.lower())))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.9.2 Determining Whom an Account Follows " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "following = []" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "paginator = tweepy.Paginator(\n", + " client.get_users_following, nasa.data.id, max_results=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for user_followed in paginator.flatten(limit=10):\n", + " following.append(user_followed.username)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('Following:', \n", + " ' '.join(sorted(following, key=lambda s: s.lower())))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.9.3 Getting a User’s Recent Tweets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nasa_tweets = client.get_users_tweets(\n", + " id=nasa.data.id, max_results=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for tweet in nasa_tweets.data:\n", + " print(f\"NASA: {tweet.data['text']}\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "jp-MarkdownHeadingCollapsed": true, + "tags": [] + }, + "source": [ + "### Grabbing Recent Tweets from Your Own Timeline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.10 Searching Recent Tweets; Intro to Twitter v2 API Search Operators" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Utility Function `print_tweets` from `tweetutilities.py`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tweetutilities import print_tweets" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```python\n", + "def print_tweets(tweets):\n", + " # translator to autodetect source language and return English\n", + " translator = GoogleTranslator(source='auto', target='en')\n", + "\n", + " \"\"\"For each tweet in tweets, display the username of the sender\n", + " and tweet text. If the language is not English, translate the text \n", + " with the deep-translator library's GoogleTranslator.\"\"\"\n", + " for tweet, user in zip(tweets.data, tweets.includes['users']):\n", + " print(f'{user.username}:', end=' ')\n", + "\n", + " if 'en' in tweet.lang:\n", + " print(f'{tweet.text}\\n')\n", + " elif 'und' not in tweet.lang: # translate to English first\n", + " print(f'\\n ORIGINAL: {tweet.text}')\n", + " print(f'TRANSLATED: {translator.translate(tweet.text)}\\n')\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching for Specific Words" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweets = client.search_recent_tweets(\n", + " query='Webb Space Telescope', \n", + " expansions=['author_id'], tweet_fields=['lang'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_tweets(tweets)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching with Twitter v2 API Search Operators" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching with Twitter v2 API Search Operators" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Operator Documentation and Tutorial" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching for Tweets From NASA Containing Links" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweets = client.search_recent_tweets(\n", + " query='from:NASA has:links', \n", + " expansions=['author_id'], tweet_fields=['lang'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_tweets(tweets)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Searching for a Hashtag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweets = client.search_recent_tweets(query='#metaverse', \n", + " expansions=['author_id'], tweet_fields=['lang'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_tweets(tweets)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.11 Spotting Trending Topics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "auth = tweepy.OAuth2BearerHandler(keys.bearer_token)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "api = tweepy.API(auth=auth, wait_on_rate_limit=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.11.1 Places with Trending Topics\n", + "**Note: This part of the Twitter APIs has not been migrated from v1.1 to v2 yet and is accessible only to \"Elevated\" and \"Academic Research\" access.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "available_trends = api.available_trends()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(available_trends)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "available_trends[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "available_trends[1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.11.2 Getting a List of Trending Topics\n", + "### Worldwide Trending Topics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "world_trends = api.get_place_trends(id=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trends_list = world_trends[0]['trends']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trends_list[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trends_list = [t for t in trends_list if t['tweet_volume']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from operator import itemgetter " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trends_list.sort(key=itemgetter('tweet_volume'), reverse=True) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for trend in trends_list:\n", + " print(trend['name'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### New York City Trending Topics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "nyc_trends = api.get_place_trends(id=2459115) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nyc_list = nyc_trends[0]['trends']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nyc_list = [t for t in nyc_list if t['tweet_volume']]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nyc_list.sort(key=itemgetter('tweet_volume'), reverse=True) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for trend in nyc_list[:5]:\n", + " print(trend['name'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.11.3 Create a Word Cloud from Trending Topics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "topics = {}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "for trend in nyc_list:\n", + " topics[trend['name']] = trend['tweet_volume']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from wordcloud import WordCloud" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wordcloud = WordCloud(width=1600, height=900,\n", + " prefer_horizontal=0.5, min_font_size=10, colormap='prism', \n", + " background_color='white') " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wordcloud = wordcloud.fit_words(topics)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wordcloud = wordcloud.to_file('TrendingTwitter.png')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**NOTE: The following code displays the image in a Jupyter Notebook**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import Image\n", + "Image(filename='TrendingTwitter.png', width=400)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "##########################################################################\n", + "# (C) Copyright 2022 by Deitel & Associates, Inc. and #\n", + "# Pearson Education, Inc. All Rights Reserved. #\n", + "# #\n", + "# DISCLAIMER: The authors and publisher of this book have used their #\n", + "# best efforts in preparing the book. These efforts include the #\n", + "# development, research, and testing of the theories and programs #\n", + "# to determine their effectiveness. The authors and publisher make #\n", + "# no warranty of any kind, expressed or implied, with regard to these #\n", + "# programs or to the documentation contained in these books. The authors #\n", + "# and publisher shall not be liable in any event for incidental or #\n", + "# consequential damages in connection with, or arising out of, the #\n", + "# furnishing, performance, or use of these programs. #\n", + "##########################################################################\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/ch12_TwitterV2/snippets_ipynb/12_12.ipynb b/examples/ch12_TwitterV2/snippets_ipynb/12_12.ipynb new file mode 100755 index 0000000..0ee9ddc --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/12_12.ipynb @@ -0,0 +1,93 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.12 Cleaning/Preprocessing Tweets for Analysis\n", + "### tweet-preprocessor Library and TextBlob Utility Functions\n", + "### Installing tweet-preprocessor\n", + "### Cleaning a Tweet" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import preprocessor as p" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "p.set_options(p.OPT.URL, p.OPT.RESERVED)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweet_text = 'RT A sample retweet with a URL https://nasa.gov'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "p.clean(tweet_text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "##########################################################################\n", + "# (C) Copyright 2022 by Deitel & Associates, Inc. and #\n", + "# Pearson Education, Inc. All Rights Reserved. #\n", + "# #\n", + "# DISCLAIMER: The authors and publisher of this book have used their #\n", + "# best efforts in preparing the book. These efforts include the #\n", + "# development, research, and testing of the theories and programs #\n", + "# to determine their effectiveness. The authors and publisher make #\n", + "# no warranty of any kind, expressed or implied, with regard to these #\n", + "# programs or to the documentation contained in these books. The authors #\n", + "# and publisher shall not be liable in any event for incidental or #\n", + "# consequential damages in connection with, or arising out of, the #\n", + "# furnishing, performance, or use of these programs. #\n", + "##########################################################################\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/ch12_TwitterV2/snippets_ipynb/12_13_02.ipynb b/examples/ch12_TwitterV2/snippets_ipynb/12_13_02.ipynb new file mode 100755 index 0000000..a42fe7b --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/12_13_02.ipynb @@ -0,0 +1,220 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.13.2 Initiating Stream Processing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Authenticating\n", + "**Don't need to authenticate in advance for streaming. Just pass bearer-token as you create the StreamingClient subclass object.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tweepy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import keys" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a `TweetListener` " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tweetlistener import TweetListener" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweet_listener = TweetListener(\n", + " bearer_token=keys.bearer_token, limit=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Redirecting Standard Error Stream to Standard Output Stream" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sys.stderr = sys.stdout" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Deleting Existing Stream Rules" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rules = tweet_listener.get_rules().data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rule_ids = [rule.id for rule in rules]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweet_listener.delete_rules(rule_ids) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating and Adding a Stream Rule" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "filter_rule = tweepy.StreamRule('football')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "tweet_listener.add_rules(filter_rule)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Starting the Tweet Stream" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweet_listener.filter(\n", + " expansions=['author_id'], tweet_fields=['lang'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Asynchronous vs. Synchronous Streams" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "##########################################################################\n", + "# (C) Copyright 2022 by Deitel & Associates, Inc. and #\n", + "# Pearson Education, Inc. All Rights Reserved. #\n", + "# #\n", + "# DISCLAIMER: The authors and publisher of this book have used their #\n", + "# best efforts in preparing the book. These efforts include the #\n", + "# development, research, and testing of the theories and programs #\n", + "# to determine their effectiveness. The authors and publisher make #\n", + "# no warranty of any kind, expressed or implied, with regard to these #\n", + "# programs or to the documentation contained in these books. The authors #\n", + "# and publisher shall not be liable in any event for incidental or #\n", + "# consequential damages in connection with, or arising out of, the #\n", + "# furnishing, performance, or use of these programs. #\n", + "##########################################################################\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/ch12_TwitterV2/snippets_ipynb/12_14.ipynb b/examples/ch12_TwitterV2/snippets_ipynb/12_14.ipynb new file mode 100644 index 0000000..a6e023a --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/12_14.ipynb @@ -0,0 +1,187 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.14 Tweet Sentiment Analysis \n", + "\n", + "**NOTE This script has been modified from what we presented in the chapter to accomodate executing it in a notebook** " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# sentimentlisener.py\n", + "\"\"\"Searches for tweets that match a search string and tallies \n", + "the number of positive, neutral and negative tweets.\"\"\"\n", + "import keys\n", + "import preprocessor as p \n", + "import sys\n", + "from textblob import TextBlob\n", + "import tweepy\n", + "\n", + "class SentimentListener(tweepy.StreamingClient):\n", + " \"\"\"Handles incoming Tweet stream.\"\"\"\n", + "\n", + " def __init__(self, bearer_token, sentiment_dict, topic, limit=10):\n", + " \"\"\"Configure the SentimentListener.\"\"\"\n", + " self.sentiment_dict = sentiment_dict\n", + " self.tweet_count = 0\n", + " self.topic = topic\n", + " self.TWEET_LIMIT = limit\n", + " \n", + " # set tweet-preprocessor to remove URLs/reserved words\n", + " p.set_options(p.OPT.URL, p.OPT.RESERVED) \n", + " super().__init__(bearer_token, wait_on_rate_limit=True)\n", + "\n", + " def on_response(self, response):\n", + " \"\"\"Called when Twitter pushes a new tweet to you.\"\"\"\n", + " \n", + " # if the tweet is not a retweet\n", + " if not response.data.text.startswith('RT'):\n", + " text = p.clean(response.data.text) # clean the tweet\n", + "\n", + " # ignore tweet if the topic is not in the tweet text\n", + " if self.topic.lower() not in text.lower():\n", + " return\n", + "\n", + " # update self.sentiment_dict with the polarity\n", + " blob = TextBlob(text)\n", + " if blob.sentiment.polarity > 0:\n", + " sentiment = '+'\n", + " self.sentiment_dict['positive'] += 1 \n", + " elif blob.sentiment.polarity == 0:\n", + " sentiment = ' '\n", + " self.sentiment_dict['neutral'] += 1 \n", + " else:\n", + " sentiment = '-'\n", + " self.sentiment_dict['negative'] += 1 \n", + "\n", + " # display the tweet\n", + " username = response.includes['users'][0].username\n", + " print(f'{sentiment} {username}: {text}\\n')\n", + "\n", + " self.tweet_count += 1 # track number of tweets processed\n", + "\n", + " # if TWEET_LIMIT is reached, terminate streaming\n", + " if self.tweet_count == self.TWEET_LIMIT:\n", + " self.disconnect()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _jubest: L Football\n", + "\n", + "+ LouG06: Thank you to @xGraciie for teddy’s first of many football vests/kits! New fan for @Biggleswadeutd #footballfriend #newfan #futureballermaybe\n", + "\n", + "+ sherlyamanda32: ◉ LIVE➠ Alma High School Vs. Shepherd High School - JuniorVarsity Football 📺📱 𝐖𝐚𝐭𝐜𝐡𝐢𝐧𝐠 𝐓𝐕 𝐋𝐢𝐧𝐤⭕► ✅ 📅 Date ➠ Wednesday, August 24, 2022 ⏰ Time ➠ 6:30 PM\n", + "\n", + "- AkshayUppal17: @amazon @AmazonBusiness @JeffBezos When Amazon India Customer service decided to play football with me. 5 agents transferring me from 1 to the other for no reason whatsoever - then ultimately calling me and putting me on hold for 12 minutes. #CustomerService #football\n", + "\n", + "+ mikemcevoy0: @KFUSCEFL feels like a rash decision surely you could have asked around for support. Feels like the tug had been pulled. Please correct me. We in the football World surely would have rallied round and supported you through your difficult times.\n", + "\n", + "+ LW_MCFC: @tiredpillbug @marcitos2k @dabelyutues @Jay63588373 @jakepaul Yeah because they have franchise sports where the players move state every 2 years 😭 there’s at least 10 countries with their own individual football leagues where the players are globally known\n", + "\n", + "+ kevinbolk: @D_ndamagi True. And it’s not like “this guy could mug me” but more like “this guy could turn a packed football stadium into a hot flesh pudding cup.”\n", + "\n", + "+ Howadlion: @btsportfootball Great legend\n", + "\n", + "- TheSono71468392: Ya grandma drank Testosterone and became a football player..and she was a Professional Tight Rope Walker Too.. ~ Neymar\n", + "\n", + "+ usc19732: 2022 Varsity Football LIVE 🏈 Spruce Creek 🆚 Seabreeze 📆 Thursday 08/25/2022 ⏰ at 7PM 📺 Live Link click here: @FalconsFlanagan @COACHRMCCREE @ZaheedPierre @demetrius_954 @FlanaganSports @Michael87501805\n", + "\n", + "Stream connection closed by Twitter\n", + "Tweet sentiment for \"football\"\n", + "Positive: 7\n", + " Neutral: 1\n", + "Negative: 2\n" + ] + } + ], + "source": [ + "#def main():\n", + "# get search term and number of tweets\n", + "search_key = 'football' #sys.argv[1]\n", + "limit = 10 #int(sys.argv[2]) # number of tweets to tally\n", + "\n", + "# set up the sentiment dictionary\n", + "sentiment_dict = {'positive': 0, 'neutral': 0, 'negative': 0}\n", + "\n", + "# create the StreamingClient subclass object\n", + "sentiment_listener = SentimentListener(keys.bearer_token, \n", + " sentiment_dict, search_key, limit)\n", + "\n", + "# redirect sys.stderr to sys.stdout\n", + "sys.stderr = sys.stdout\n", + "\n", + "# delete existing stream rules\n", + "rules = sentiment_listener.get_rules().data\n", + "rule_ids = [rule.id for rule in rules]\n", + "sentiment_listener.delete_rules(rule_ids) \n", + "\n", + "# create stream rule\n", + "sentiment_listener.add_rules(\n", + " tweepy.StreamRule(f'{search_key} lang:en'))\n", + "\n", + "# start filtering English tweets containing search_key\n", + "sentiment_listener.filter(expansions=['author_id'])\n", + "\n", + "print(f'Tweet sentiment for \"{search_key}\"')\n", + "print('Positive:', sentiment_dict['positive'])\n", + "print(' Neutral:', sentiment_dict['neutral'])\n", + "print('Negative:', sentiment_dict['negative'])\n", + "\n", + "# call main if this file is executed as a script\n", + "#if __name__ == '__main__':\n", + "# main()\n", + "\n", + "##########################################################################\n", + "# (C) Copyright 2022 by Deitel & Associates, Inc. and #\n", + "# Pearson Education, Inc. All Rights Reserved. #\n", + "# #\n", + "# DISCLAIMER: The authors and publisher of this book have used their #\n", + "# best efforts in preparing the book. These efforts include the #\n", + "# development, research, and testing of the theories and programs #\n", + "# to determine their effectiveness. The authors and publisher make #\n", + "# no warranty of any kind, expressed or implied, with regard to these #\n", + "# programs or to the documentation contained in these books. The authors #\n", + "# and publisher shall not be liable in any event for incidental or #\n", + "# consequential damages in connection with, or arising out of, the #\n", + "# furnishing, performance, or use of these programs. #\n", + "##########################################################################" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/ch12_TwitterV2/snippets_ipynb/12_15_01.ipynb b/examples/ch12_TwitterV2/snippets_ipynb/12_15_01.ipynb new file mode 100755 index 0000000..e38c502 --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/12_15_01.ipynb @@ -0,0 +1,420 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 12.15.1 Getting and Mapping the Tweets" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Collections Required By `LocationListener`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tweets = [] " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "counts = {'total_tweets': 0, 'locations': 0}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating the `LocationListener` " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import keys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tweepy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from locationlistener import LocationListener" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "location_listener = LocationListener(\n", + " keys.bearer_token, counts_dict=counts, tweets_list=tweets,\n", + " topic='football', limit=50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Redirect sys.stderr to sys.stdout" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sys.stderr = sys.stdout" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Delete Existing `StreamRule`s" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rules = location_listener.get_rules().data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "rule_ids = [rule.id for rule in rules]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "location_listener.delete_rules(rule_ids) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a `StreamRule`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "location_listener.add_rules(\n", + " tweepy.StreamRule('football lang:en'))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### Configure and Start the Stream of Tweets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "location_listener.filter(expansions=['author_id'], \n", + " user_fields=['location'], tweet_fields=['lang'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Displaying the Location Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "counts['total_tweets']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "counts['locations']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f'{counts[\"locations\"] / counts[\"total_tweets\"]:.1%}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Geocoding the Locations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tweetutilities import get_geocodes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bad_locations = get_geocodes(tweets)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Displaying the Bad Location Statistics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bad_locations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(f'{bad_locations / counts[\"locations\"]:.1%}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Cleaning the Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(tweets)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = df.dropna()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating a Map with Folium" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import folium" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "usmap = folium.Map(location=[39.8283, -98.5795], \n", + " tiles='Stamen Terrain', zoom_start=5, detect_retina=True) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Creating Popup Markers for the Tweet Locations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for t in df.itertuples():\n", + " text = ': '.join([t.username, t.text])\n", + " popup = folium.Popup(text, parse_html=True)\n", + " marker = folium.Marker((t.latitude, t.longitude), \n", + " popup=popup)\n", + " marker.add_to(usmap)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Saving the Map" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "usmap.save('tweet_map.html')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**NOTE: We added the following cell to display the map in the Jupyter Notebook.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "usmap" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "##########################################################################\n", + "# (C) Copyright 2022 by Deitel & Associates, Inc. and #\n", + "# Pearson Education, Inc. All Rights Reserved. #\n", + "# #\n", + "# DISCLAIMER: The authors and publisher of this book have used their #\n", + "# best efforts in preparing the book. These efforts include the #\n", + "# development, research, and testing of the theories and programs #\n", + "# to determine their effectiveness. The authors and publisher make #\n", + "# no warranty of any kind, expressed or implied, with regard to these #\n", + "# programs or to the documentation contained in these books. The authors #\n", + "# and publisher shall not be liable in any event for incidental or #\n", + "# consequential damages in connection with, or arising out of, the #\n", + "# furnishing, performance, or use of these programs. #\n", + "##########################################################################\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/ch12_TwitterV2/snippets_ipynb/README.txt b/examples/ch12_TwitterV2/snippets_ipynb/README.txt new file mode 100755 index 0000000..ac3297c --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/README.txt @@ -0,0 +1,2 @@ +Sections 13.7-11 use one continuous IPython session for the examples +and Self Check exercises, so all of these are in one notebook. diff --git a/examples/ch12_TwitterV2/snippets_ipynb/files/art/.ipynb_checkpoints/check-checkpoint.png b/examples/ch12_TwitterV2/snippets_ipynb/files/art/.ipynb_checkpoints/check-checkpoint.png new file mode 100755 index 0000000..eea18cd Binary files /dev/null and b/examples/ch12_TwitterV2/snippets_ipynb/files/art/.ipynb_checkpoints/check-checkpoint.png differ diff --git a/examples/ch12_TwitterV2/snippets_ipynb/files/art/check.png b/examples/ch12_TwitterV2/snippets_ipynb/files/art/check.png new file mode 100755 index 0000000..eea18cd Binary files /dev/null and b/examples/ch12_TwitterV2/snippets_ipynb/files/art/check.png differ diff --git a/examples/ch12_TwitterV2/snippets_ipynb/keys.py b/examples/ch12_TwitterV2/snippets_ipynb/keys.py new file mode 100644 index 0000000..9fcffad --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/keys.py @@ -0,0 +1,8 @@ +#consumer_key = 'NVqJ06iaqBbyIQDuxUUCiM5gd' +#consumer_secret = 'fyVLUOxYMH6GSy8x7aA1OKLfhqUeRst4bnRMTImN1ASym0cgI9' +#access_token = '24867870-4rhn9y3OzIK9OBu57QMBVHQWKZfMxWRxPZTJIIsKX' +#access_token_secret = '3lv07LC30CKyzOjIEJiK1P5pdk94KIRTduFPqnIBU8LXb' + +mapquest_key = 'L0xOBy7HqIWsNgBvZvgVZx9HX1GSYzri' +#deep_translator_key = 'de06403b5fda5f896b9b7c23df3e7c0a' +bearer_token = 'AAAAAAAAAAAAAAAAAAAAAKKybwEAAAAAzvQFdKaTNAYKLzB1RX7Pmz25yJg%3DgNB7yVp00rLtwsmETSvoePoc2ymdOYfZb54Mw8A4LqwpYlTC2X' \ No newline at end of file diff --git a/examples/ch12_TwitterV2/snippets_ipynb/keys_empty.py b/examples/ch12_TwitterV2/snippets_ipynb/keys_empty.py new file mode 100755 index 0000000..bf7c356 --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/keys_empty.py @@ -0,0 +1,2 @@ +bearer_token = 'YourBearerToken' +mapquest_key = 'YourAPIKey' \ No newline at end of file diff --git a/examples/ch12_TwitterV2/snippets_ipynb/locationlistener.py b/examples/ch12_TwitterV2/snippets_ipynb/locationlistener.py new file mode 100755 index 0000000..f154d61 --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/locationlistener.py @@ -0,0 +1,59 @@ +# locationlistener.py +"""Receives tweets matching a search string and stores a list of +dictionaries containing each tweet's username/text/location.""" +import tweepy +from tweetutilities import get_tweet_content + +class LocationListener(tweepy.StreamingClient): + """Handles incoming Tweet stream to get location data.""" + + def __init__(self, bearer_token, counts_dict, + tweets_list, topic, limit=10): + """Configure the LocationListener.""" + self.tweets_list = tweets_list + self.counts_dict = counts_dict + self.topic = topic + self.TWEET_LIMIT = limit + super().__init__(bearer_token, wait_on_rate_limit=True) + + def on_response(self, response): + """Called when Twitter pushes a new tweet to you.""" + + # get tweet's username, text and location + tweet_data = get_tweet_content(response) + + # ignore retweets and tweets that do not contain the topic + if (tweet_data['text'].startswith('RT') or + self.topic.lower() not in tweet_data['text'].lower()): + return + + self.counts_dict['total_tweets'] += 1 # it's an original tweet + + # ignore tweets with no location + if not tweet_data.get('location'): + return + + self.counts_dict['locations'] += 1 # user account has location + self.tweets_list.append(tweet_data) # store the tweet + print(f"{tweet_data['username']}: {tweet_data['text']}\n") + + # if TWEET_LIMIT is reached, terminate streaming + if self.counts_dict['locations'] == self.TWEET_LIMIT: + self.disconnect() + + + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_ipynb/sentimentlistener.py b/examples/ch12_TwitterV2/snippets_ipynb/sentimentlistener.py new file mode 100755 index 0000000..2cc45ec --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/sentimentlistener.py @@ -0,0 +1,106 @@ +# sentimentlisener.py +"""Searches for tweets that match a search string and tallies +the number of positive, neutral and negative tweets.""" +import keys +import preprocessor as p +import sys +from textblob import TextBlob +import tweepy + +class SentimentListener(tweepy.StreamingClient): + """Handles incoming Tweet stream.""" + + def __init__(self, bearer_token, sentiment_dict, topic, limit=10): + """Configure the SentimentListener.""" + self.sentiment_dict = sentiment_dict + self.tweet_count = 0 + self.topic = topic + self.TWEET_LIMIT = limit + + # set tweet-preprocessor to remove URLs/reserved words + p.set_options(p.OPT.URL, p.OPT.RESERVED) + super().__init__(bearer_token, wait_on_rate_limit=True) + + def on_response(self, response): + """Called when Twitter pushes a new tweet to you.""" + + # if the tweet is not a retweet + if not response.data.text.startswith('RT'): + text = p.clean(response.data.text) # clean the tweet + + # ignore tweet if the topic is not in the tweet text + if self.topic.lower() not in text.lower(): + return + + # update self.sentiment_dict with the polarity + blob = TextBlob(text) + if blob.sentiment.polarity > 0: + sentiment = '+' + self.sentiment_dict['positive'] += 1 + elif blob.sentiment.polarity == 0: + sentiment = ' ' + self.sentiment_dict['neutral'] += 1 + else: + sentiment = '-' + self.sentiment_dict['negative'] += 1 + + # display the tweet + username = response.includes['users'][0].username + print(f'{sentiment} {username}: {text}\n') + + self.tweet_count += 1 # track number of tweets processed + + # if TWEET_LIMIT is reached, terminate streaming + if self.tweet_count == self.TWEET_LIMIT: + self.disconnect() + +def main(): + # get search term and number of tweets + search_key = sys.argv[1] + limit = int(sys.argv[2]) # number of tweets to tally + + # set up the sentiment dictionary + sentiment_dict = {'positive': 0, 'neutral': 0, 'negative': 0} + + # create the StreamingClient subclass object + sentiment_listener = SentimentListener(keys.bearer_token, + sentiment_dict, search_key, limit) + + # redirect sys.stderr to sys.stdout + sys.stderr = sys.stdout + + # delete existing stream rules + rules = sentiment_listener.get_rules().data + rule_ids = [rule.id for rule in rules] + sentiment_listener.delete_rules(rule_ids) + + # create stream rule + sentiment_listener.add_rules( + tweepy.StreamRule(f'{search_key} lang:en')) + + # start filtering English tweets containing search_key + sentiment_listener.filter(expansions=['author_id']) + + print(f'Tweet sentiment for "{search_key}"') + print('Positive:', sentiment_dict['positive']) + print(' Neutral:', sentiment_dict['neutral']) + print('Negative:', sentiment_dict['negative']) + +# call main if this file is executed as a script +if __name__ == '__main__': + main() + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_ipynb/tweetlistener.py b/examples/ch12_TwitterV2/snippets_ipynb/tweetlistener.py new file mode 100755 index 0000000..b01654d --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/tweetlistener.py @@ -0,0 +1,61 @@ +# tweetlistener.py +"""StreamListener subclass that processes tweets as they arrive.""" +from deep_translator import GoogleTranslator +import tweepy + +class TweetListener(tweepy.StreamingClient): + """Handles incoming Tweet stream.""" + + def __init__(self, bearer_token, limit=10): + """Create instance variables for tracking number of tweets.""" + self.tweet_count = 0 + self.TWEET_LIMIT = limit + + # GoogleTranslator object for translating tweets to English + self.translator = GoogleTranslator(source='auto', target='en') + + super().__init__(bearer_token, wait_on_rate_limit=True) + + def on_connect(self): + """Called when your connection attempt is successful, enabling + you to perform appropriate application tasks at that point.""" + print('Connection successful\n') + + def on_response(self, response): + """Called when Twitter pushes a new tweet to you.""" + + try: + # get username of user who sent the tweet + username = response.includes['users'][0].username + print(f'Screen name: {username}') + print(f' Language: {response.data.lang}') + print(f' Tweet text: {response.data.text}') + + if response.data.lang != 'en' and response.data.lang != 'und': + english = self.translator.translate(response.data.text) + print(f' Translated: {english}') + + print() + self.tweet_count += 1 + except Exception as e: + print(f'Exception occured: {e}') + self.disconnect() + + # if TWEET_LIMIT is reached, terminate streaming + if self.tweet_count == self.TWEET_LIMIT: + self.disconnect() + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_ipynb/tweetutilities.py b/examples/ch12_TwitterV2/snippets_ipynb/tweetutilities.py new file mode 100755 index 0000000..f5a12ab --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_ipynb/tweetutilities.py @@ -0,0 +1,76 @@ +# tweetutilities.py +"""Utility functions for interacting with Tweepy objects.""" +from deep_translator import GoogleTranslator +from geopy import OpenMapQuest +import keys +import time +import tweepy + +def print_tweets(tweets): + # translator to autodetect source language and return English + translator = GoogleTranslator(source='auto', target='en') + + """For each tweet in tweets, display the username of the sender + and tweet text. If the language is not English, translate the text + with Deep Translator.""" + for tweet, user in zip(tweets.data, tweets.includes['users']): + print(f'{user.username}:', end=' ') + + if 'en' in tweet.lang: + print(f'{tweet.text}\n') + elif 'und' not in tweet.lang: # translate to English first + print(f'\n ORIGINAL: {tweet.text}') + print(f'TRANSLATED: {translator.translate(tweet.text)}\n') + +def get_tweet_content(response): + """Return dictionary with data from tweet.""" + fields = {} + fields['username'] = response.includes['users'][0].username + fields['text'] = response.data.text + fields['location'] = response.includes['users'][0].location + + return fields + +def get_geocodes(tweet_list): + """Get the latitude and longitude for each tweet's location. + Returns the number of tweets with invalid location data.""" + print('Getting coordinates for tweet locations...') + geo = OpenMapQuest(api_key=keys.mapquest_key) # geocoder + bad_locations = 0 + + for tweet in tweet_list: + processed = False + delay = .1 # used if OpenMapQuest times out to delay next call + while not processed: + try: # get coordinates for tweet['location'] + geo_location = geo.geocode(tweet['location']) + processed = True + except: # timed out, so wait before trying again + print('OpenMapQuest service timed out. Waiting.') + time.sleep(delay) + delay += .1 + + if geo_location: + tweet['latitude'] = geo_location.latitude + tweet['longitude'] = geo_location.longitude + else: + bad_locations += 1 # tweet['location'] was invalid + + print('Done geocoding') + return bad_locations + + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_py/12_07-11.py b/examples/ch12_TwitterV2/snippets_py/12_07-11.py new file mode 100755 index 0000000..bdbb44e --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/12_07-11.py @@ -0,0 +1,172 @@ +# Section 12.7-12.11 snippets because those sections are one running IPython session + +# 12.7 Authenticating with Twitter Via Tweepy to Access Twitter v2 APIs +import tweepy +import keys + +# Creating a Client Object +client = tweepy.Client(bearer_token=keys.bearer_token, + wait_on_rate_limit=True) + +# 12.8 Getting Information About a Twitter Account +nasa = client.get_user(username='NASA', + user_fields=['description', 'public_metrics']) + +# tweepy.Response Object +# Getting a User’s Basic Account Information +nasa.data.id + +nasa.data.name + +nasa.data.username + +nasa.data.description + +# Getting the Number of Accounts That Follow This Account and the Number of Accounts This Account Follows +nasa.data.public_metrics['followers_count'] + +nasa.data.public_metrics['following_count'] + +# Getting Your Own Account’s Information + +# 12.9 Intro to Tweepy Paginators: Getting More than One Page of Results +# 12.9.1 Determining an Account’s Followers +followers = [] + +# Creating a Paginator +paginator = tweepy.Paginator( + client.get_users_followers, nasa.data.id, max_results=5) + +# Getting Results +for follower in paginator.flatten(limit=10): + followers.append(follower.username) + +print('Followers:', + ' '.join(sorted(followers, key=lambda s: s.lower()))) + +# 12.9.2 Determining Whom an Account Follows +following = [] + +paginator = tweepy.Paginator( + client.get_users_following, nasa.data.id, max_results=5) + +for user_followed in paginator.flatten(limit=10): + following.append(user_followed.username) + +print('Following:', + ' '.join(sorted(following, key=lambda s: s.lower()))) + + +# 12.9.3 Getting a User’s Recent Tweets +nasa_tweets = client.get_users_tweets( + id=nasa.data.id, max_results=5) + +for tweet in nasa_tweets.data: + print(f"NASA: {tweet.data['text']}\n") + +# Grabbing Recent Tweets from Your Own Timeline + +# 12.10 Searching Recent Tweets; Intro to Twitter v2 API Search Operators +# Utility Function print_tweets from tweetutilities.py +from tweetutilities import print_tweets + +# Searching for Specific Words +tweets = client.search_recent_tweets( + query='Webb Space Telescope', + expansions=['author_id'], tweet_fields=['lang']) + +print_tweets(tweets) + +# Searching with Twitter v2 API Search Operators + +# Operator Documentation and Tutorial + +# Searching for Tweets From NASA Containing Links +tweets = client.search_recent_tweets( + query='from:NASA has:links', + expansions=['author_id'], tweet_fields=['lang']) + +print_tweets(tweets) + +# Searching for a Hashtag +tweets = client.search_recent_tweets(query='#metaverse', + expansions=['author_id'], tweet_fields=['lang']) + +print_tweets(tweets) + +# 12.11 Spotting Trending Topics +auth = tweepy.OAuth2BearerHandler(keys.bearer_token) + +api = tweepy.API(auth=auth, wait_on_rate_limit=True) + +# 12.11.1 Places with Trending Topics +# Note: This part of the Twitter APIs has not been migrated from v1.1 to v2 +# yet and is accessible only to "Elevated" and "Academic Research" access. +available_trends = api.available_trends() + +len(available_trends) + +available_trends[0] + +available_trends[1] + +# 12.11.2 Getting a List of Trending Topics +# Worldwide Trending Topics +world_trends = api.get_place_trends(id=1) + +trends_list = world_trends[0]['trends'] + +trends_list[0] + +trends_list = [t for t in trends_list if t['tweet_volume']] + +from operator import itemgetter +trends_list.sort(key=itemgetter('tweet_volume'), reverse=True) + +for trend in trends_list: + print(trend['name']) + +# New York City Trending Topics +nyc_trends = api.get_place_trends(id=2459115) + +nyc_list = nyc_trends[0]['trends'] + +nyc_list = [t for t in nyc_list if t['tweet_volume']] + +nyc_list.sort(key=itemgetter('tweet_volume'), reverse=True) + +for trend in nyc_list[:5]: + print(trend['name']) + +# 12.11.3 Create a Word Cloud from Trending Topics +topics = {} + +for trend in nyc_list: + topics[trend['name']] = trend['tweet_volume'] + +from wordcloud import WordCloud + +wordcloud = WordCloud(width=1600, height=900, + prefer_horizontal=0.5, min_font_size=10, colormap='prism', + background_color='white') + +wordcloud = wordcloud.fit_words(topics) + +wordcloud = wordcloud.to_file('TrendingTwitter.png') + + + +########################################################################## +# (C) Copyright 2019 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_py/12_12.py b/examples/ch12_TwitterV2/snippets_py/12_12.py new file mode 100755 index 0000000..f627440 --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/12_12.py @@ -0,0 +1,28 @@ +# Section 12.12 snippets + +import preprocessor as p + +p.set_options(p.OPT.URL, p.OPT.RESERVED) + +tweet_text = 'RT A sample retweet with a URL https://nasa.gov' + +p.clean(tweet_text) + + + + + +########################################################################## +# (C) Copyright 2019 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_py/12_13_02.py b/examples/ch12_TwitterV2/snippets_py/12_13_02.py new file mode 100755 index 0000000..b0d861b --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/12_13_02.py @@ -0,0 +1,52 @@ +# 12.13.2 Initiating Stream Processing + +# Authenticating +import tweepy + +import keys + +# Creating a TweetListener +from tweetlistener import TweetListener + +tweet_listener = TweetListener( + bearer_token=keys.bearer_token, limit=3) + +# Redirecting Standard Error Stream to Standard Output Stream +import sys + +sys.stderr = sys.stdout + +# Deleting Existing Stream Rules +rules = tweet_listener.get_rules().data + +rule_ids = [rule.id for rule in rules] + +tweet_listener.delete_rules(rule_ids) + +# Creating and Adding a Stream Rule +filter_rule = tweepy.StreamRule('football') + +tweet_listener.add_rules(filter_rule) + +# Starting the Tweet Stream +tweet_listener.filter( + expansions=['author_id'], tweet_fields=['lang']) + +# Stream connection closed by Twitter + +# Asynchronous vs. Synchronous Streams + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_py/12_15_01.py b/examples/ch12_TwitterV2/snippets_py/12_15_01.py new file mode 100755 index 0000000..ad1afe4 --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/12_15_01.py @@ -0,0 +1,103 @@ +# 12.15.1 Getting and Mapping the Tweets + +# Collections Required By LocationListener +tweets = [] + +counts = {'total_tweets': 0, 'locations': 0} + +# Creating the LocationListener +import keys + +import tweepy + +from locationlistener import LocationListener + +location_listener = LocationListener( + keys.bearer_token, counts_dict=counts, tweets_list=tweets, + topic='football', limit=50) + +# Redirect sys.stderr to sys.stdout +import sys + +sys.stderr = sys.stdout + +# Delete Existing StreamRules +rules = location_listener.get_rules().data + +rule_ids = [rule.id for rule in rules] + +location_listener.delete_rules(rule_ids) + +# Create a StreamRule +location_listener.add_rules( + tweepy.StreamRule('football lang:en')) + +# Configure and Start the Stream of Tweets +location_listener.filter(expansions=['author_id'], + user_fields=['location'], tweet_fields=['lang']) + +# Displaying the Location Statistics +counts['total_tweets'] + +counts['locations'] + +print(f'{counts["locations"] / counts["total_tweets"]:.1%}') + +# Geocoding the Locations +from tweetutilities import get_geocodes + +bad_locations = get_geocodes(tweets) + +# Displaying the Bad Location Statistics +bad_locations + +print(f'{bad_locations / counts["locations"]:.1%}') + +# Cleaning the Data +import pandas as pd + +df = pd.DataFrame(tweets) + +df = df.dropna() + +# Creating a Map with Folium +import folium + +usmap = folium.Map(location=[39.8283, -98.5795], + tiles='Stamen Terrain', zoom_start=5, detect_retina=True) + +# Creating Popup Markers for the Tweet Locations +for t in df.itertuples(): + text = ': '.join([t.username, t.text]) + popup = folium.Popup(text, parse_html=True) + marker = folium.Marker((t.latitude, t.longitude), + popup=popup) + marker.add_to(usmap) + +# Saving the Map +usmap.save('tweet_map.html') + + + + + + + + + + + +########################################################################## +# (C) Copyright 2019 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_py/README.txt b/examples/ch12_TwitterV2/snippets_py/README.txt new file mode 100755 index 0000000..3fe48c8 --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/README.txt @@ -0,0 +1,3 @@ +Sections 13.7-11 use one continuous IPython session for the +examples and Self Check exercises, so all of these are in one +snippet file. diff --git a/examples/ch12_TwitterV2/snippets_py/keys.py b/examples/ch12_TwitterV2/snippets_py/keys.py new file mode 100755 index 0000000..bf7c356 --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/keys.py @@ -0,0 +1,2 @@ +bearer_token = 'YourBearerToken' +mapquest_key = 'YourAPIKey' \ No newline at end of file diff --git a/examples/ch12_TwitterV2/snippets_py/locationlistener.py b/examples/ch12_TwitterV2/snippets_py/locationlistener.py new file mode 100755 index 0000000..f154d61 --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/locationlistener.py @@ -0,0 +1,59 @@ +# locationlistener.py +"""Receives tweets matching a search string and stores a list of +dictionaries containing each tweet's username/text/location.""" +import tweepy +from tweetutilities import get_tweet_content + +class LocationListener(tweepy.StreamingClient): + """Handles incoming Tweet stream to get location data.""" + + def __init__(self, bearer_token, counts_dict, + tweets_list, topic, limit=10): + """Configure the LocationListener.""" + self.tweets_list = tweets_list + self.counts_dict = counts_dict + self.topic = topic + self.TWEET_LIMIT = limit + super().__init__(bearer_token, wait_on_rate_limit=True) + + def on_response(self, response): + """Called when Twitter pushes a new tweet to you.""" + + # get tweet's username, text and location + tweet_data = get_tweet_content(response) + + # ignore retweets and tweets that do not contain the topic + if (tweet_data['text'].startswith('RT') or + self.topic.lower() not in tweet_data['text'].lower()): + return + + self.counts_dict['total_tweets'] += 1 # it's an original tweet + + # ignore tweets with no location + if not tweet_data.get('location'): + return + + self.counts_dict['locations'] += 1 # user account has location + self.tweets_list.append(tweet_data) # store the tweet + print(f"{tweet_data['username']}: {tweet_data['text']}\n") + + # if TWEET_LIMIT is reached, terminate streaming + if self.counts_dict['locations'] == self.TWEET_LIMIT: + self.disconnect() + + + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_py/sentimentlistener.py b/examples/ch12_TwitterV2/snippets_py/sentimentlistener.py new file mode 100755 index 0000000..2cc45ec --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/sentimentlistener.py @@ -0,0 +1,106 @@ +# sentimentlisener.py +"""Searches for tweets that match a search string and tallies +the number of positive, neutral and negative tweets.""" +import keys +import preprocessor as p +import sys +from textblob import TextBlob +import tweepy + +class SentimentListener(tweepy.StreamingClient): + """Handles incoming Tweet stream.""" + + def __init__(self, bearer_token, sentiment_dict, topic, limit=10): + """Configure the SentimentListener.""" + self.sentiment_dict = sentiment_dict + self.tweet_count = 0 + self.topic = topic + self.TWEET_LIMIT = limit + + # set tweet-preprocessor to remove URLs/reserved words + p.set_options(p.OPT.URL, p.OPT.RESERVED) + super().__init__(bearer_token, wait_on_rate_limit=True) + + def on_response(self, response): + """Called when Twitter pushes a new tweet to you.""" + + # if the tweet is not a retweet + if not response.data.text.startswith('RT'): + text = p.clean(response.data.text) # clean the tweet + + # ignore tweet if the topic is not in the tweet text + if self.topic.lower() not in text.lower(): + return + + # update self.sentiment_dict with the polarity + blob = TextBlob(text) + if blob.sentiment.polarity > 0: + sentiment = '+' + self.sentiment_dict['positive'] += 1 + elif blob.sentiment.polarity == 0: + sentiment = ' ' + self.sentiment_dict['neutral'] += 1 + else: + sentiment = '-' + self.sentiment_dict['negative'] += 1 + + # display the tweet + username = response.includes['users'][0].username + print(f'{sentiment} {username}: {text}\n') + + self.tweet_count += 1 # track number of tweets processed + + # if TWEET_LIMIT is reached, terminate streaming + if self.tweet_count == self.TWEET_LIMIT: + self.disconnect() + +def main(): + # get search term and number of tweets + search_key = sys.argv[1] + limit = int(sys.argv[2]) # number of tweets to tally + + # set up the sentiment dictionary + sentiment_dict = {'positive': 0, 'neutral': 0, 'negative': 0} + + # create the StreamingClient subclass object + sentiment_listener = SentimentListener(keys.bearer_token, + sentiment_dict, search_key, limit) + + # redirect sys.stderr to sys.stdout + sys.stderr = sys.stdout + + # delete existing stream rules + rules = sentiment_listener.get_rules().data + rule_ids = [rule.id for rule in rules] + sentiment_listener.delete_rules(rule_ids) + + # create stream rule + sentiment_listener.add_rules( + tweepy.StreamRule(f'{search_key} lang:en')) + + # start filtering English tweets containing search_key + sentiment_listener.filter(expansions=['author_id']) + + print(f'Tweet sentiment for "{search_key}"') + print('Positive:', sentiment_dict['positive']) + print(' Neutral:', sentiment_dict['neutral']) + print('Negative:', sentiment_dict['negative']) + +# call main if this file is executed as a script +if __name__ == '__main__': + main() + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_py/tweetlistener.py b/examples/ch12_TwitterV2/snippets_py/tweetlistener.py new file mode 100755 index 0000000..b01654d --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/tweetlistener.py @@ -0,0 +1,61 @@ +# tweetlistener.py +"""StreamListener subclass that processes tweets as they arrive.""" +from deep_translator import GoogleTranslator +import tweepy + +class TweetListener(tweepy.StreamingClient): + """Handles incoming Tweet stream.""" + + def __init__(self, bearer_token, limit=10): + """Create instance variables for tracking number of tweets.""" + self.tweet_count = 0 + self.TWEET_LIMIT = limit + + # GoogleTranslator object for translating tweets to English + self.translator = GoogleTranslator(source='auto', target='en') + + super().__init__(bearer_token, wait_on_rate_limit=True) + + def on_connect(self): + """Called when your connection attempt is successful, enabling + you to perform appropriate application tasks at that point.""" + print('Connection successful\n') + + def on_response(self, response): + """Called when Twitter pushes a new tweet to you.""" + + try: + # get username of user who sent the tweet + username = response.includes['users'][0].username + print(f'Screen name: {username}') + print(f' Language: {response.data.lang}') + print(f' Tweet text: {response.data.text}') + + if response.data.lang != 'en' and response.data.lang != 'und': + english = self.translator.translate(response.data.text) + print(f' Translated: {english}') + + print() + self.tweet_count += 1 + except Exception as e: + print(f'Exception occured: {e}') + self.disconnect() + + # if TWEET_LIMIT is reached, terminate streaming + if self.tweet_count == self.TWEET_LIMIT: + self.disconnect() + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/snippets_py/tweetutilities.py b/examples/ch12_TwitterV2/snippets_py/tweetutilities.py new file mode 100755 index 0000000..f5a12ab --- /dev/null +++ b/examples/ch12_TwitterV2/snippets_py/tweetutilities.py @@ -0,0 +1,76 @@ +# tweetutilities.py +"""Utility functions for interacting with Tweepy objects.""" +from deep_translator import GoogleTranslator +from geopy import OpenMapQuest +import keys +import time +import tweepy + +def print_tweets(tweets): + # translator to autodetect source language and return English + translator = GoogleTranslator(source='auto', target='en') + + """For each tweet in tweets, display the username of the sender + and tweet text. If the language is not English, translate the text + with Deep Translator.""" + for tweet, user in zip(tweets.data, tweets.includes['users']): + print(f'{user.username}:', end=' ') + + if 'en' in tweet.lang: + print(f'{tweet.text}\n') + elif 'und' not in tweet.lang: # translate to English first + print(f'\n ORIGINAL: {tweet.text}') + print(f'TRANSLATED: {translator.translate(tweet.text)}\n') + +def get_tweet_content(response): + """Return dictionary with data from tweet.""" + fields = {} + fields['username'] = response.includes['users'][0].username + fields['text'] = response.data.text + fields['location'] = response.includes['users'][0].location + + return fields + +def get_geocodes(tweet_list): + """Get the latitude and longitude for each tweet's location. + Returns the number of tweets with invalid location data.""" + print('Getting coordinates for tweet locations...') + geo = OpenMapQuest(api_key=keys.mapquest_key) # geocoder + bad_locations = 0 + + for tweet in tweet_list: + processed = False + delay = .1 # used if OpenMapQuest times out to delay next call + while not processed: + try: # get coordinates for tweet['location'] + geo_location = geo.geocode(tweet['location']) + processed = True + except: # timed out, so wait before trying again + print('OpenMapQuest service timed out. Waiting.') + time.sleep(delay) + delay += .1 + + if geo_location: + tweet['latitude'] = geo_location.latitude + tweet['longitude'] = geo_location.longitude + else: + bad_locations += 1 # tweet['location'] was invalid + + print('Done geocoding') + return bad_locations + + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/tweetlistener.py b/examples/ch12_TwitterV2/tweetlistener.py new file mode 100755 index 0000000..b01654d --- /dev/null +++ b/examples/ch12_TwitterV2/tweetlistener.py @@ -0,0 +1,61 @@ +# tweetlistener.py +"""StreamListener subclass that processes tweets as they arrive.""" +from deep_translator import GoogleTranslator +import tweepy + +class TweetListener(tweepy.StreamingClient): + """Handles incoming Tweet stream.""" + + def __init__(self, bearer_token, limit=10): + """Create instance variables for tracking number of tweets.""" + self.tweet_count = 0 + self.TWEET_LIMIT = limit + + # GoogleTranslator object for translating tweets to English + self.translator = GoogleTranslator(source='auto', target='en') + + super().__init__(bearer_token, wait_on_rate_limit=True) + + def on_connect(self): + """Called when your connection attempt is successful, enabling + you to perform appropriate application tasks at that point.""" + print('Connection successful\n') + + def on_response(self, response): + """Called when Twitter pushes a new tweet to you.""" + + try: + # get username of user who sent the tweet + username = response.includes['users'][0].username + print(f'Screen name: {username}') + print(f' Language: {response.data.lang}') + print(f' Tweet text: {response.data.text}') + + if response.data.lang != 'en' and response.data.lang != 'und': + english = self.translator.translate(response.data.text) + print(f' Translated: {english}') + + print() + self.tweet_count += 1 + except Exception as e: + print(f'Exception occured: {e}') + self.disconnect() + + # if TWEET_LIMIT is reached, terminate streaming + if self.tweet_count == self.TWEET_LIMIT: + self.disconnect() + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +########################################################################## diff --git a/examples/ch12_TwitterV2/tweetutilities.py b/examples/ch12_TwitterV2/tweetutilities.py new file mode 100755 index 0000000..f5a12ab --- /dev/null +++ b/examples/ch12_TwitterV2/tweetutilities.py @@ -0,0 +1,76 @@ +# tweetutilities.py +"""Utility functions for interacting with Tweepy objects.""" +from deep_translator import GoogleTranslator +from geopy import OpenMapQuest +import keys +import time +import tweepy + +def print_tweets(tweets): + # translator to autodetect source language and return English + translator = GoogleTranslator(source='auto', target='en') + + """For each tweet in tweets, display the username of the sender + and tweet text. If the language is not English, translate the text + with Deep Translator.""" + for tweet, user in zip(tweets.data, tweets.includes['users']): + print(f'{user.username}:', end=' ') + + if 'en' in tweet.lang: + print(f'{tweet.text}\n') + elif 'und' not in tweet.lang: # translate to English first + print(f'\n ORIGINAL: {tweet.text}') + print(f'TRANSLATED: {translator.translate(tweet.text)}\n') + +def get_tweet_content(response): + """Return dictionary with data from tweet.""" + fields = {} + fields['username'] = response.includes['users'][0].username + fields['text'] = response.data.text + fields['location'] = response.includes['users'][0].location + + return fields + +def get_geocodes(tweet_list): + """Get the latitude and longitude for each tweet's location. + Returns the number of tweets with invalid location data.""" + print('Getting coordinates for tweet locations...') + geo = OpenMapQuest(api_key=keys.mapquest_key) # geocoder + bad_locations = 0 + + for tweet in tweet_list: + processed = False + delay = .1 # used if OpenMapQuest times out to delay next call + while not processed: + try: # get coordinates for tweet['location'] + geo_location = geo.geocode(tweet['location']) + processed = True + except: # timed out, so wait before trying again + print('OpenMapQuest service timed out. Waiting.') + time.sleep(delay) + delay += .1 + + if geo_location: + tweet['latitude'] = geo_location.latitude + tweet['longitude'] = geo_location.longitude + else: + bad_locations += 1 # tweet['location'] was invalid + + print('Done geocoding') + return bad_locations + + +########################################################################## +# (C) Copyright 2022 by Deitel & Associates, Inc. and # +# Pearson Education, Inc. All Rights Reserved. # +# # +# DISCLAIMER: The authors and publisher of this book have used their # +# best efforts in preparing the book. These efforts include the # +# development, research, and testing of the theories and programs # +# to determine their effectiveness. The authors and publisher make # +# no warranty of any kind, expressed or implied, with regard to these # +# programs or to the documentation contained in these books. The authors # +# and publisher shall not be liable in any event for incidental or # +# consequential damages in connection with, or arising out of, the # +# furnishing, performance, or use of these programs. # +##########################################################################