{

"cells": [
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"# Unit-2\n"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## Data Loading"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Loading the dataset\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### importing the data the mail data\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"X-GM-THRID\n",
"X-Gmail-Labels\n",
"MIME-Version\n",
"Date\n",
"Message-ID\n",
 "Subject\n",
"From\n",
"To\n",
"Content-Type\n"
]
},
{
"data": {
"text/plain": [
"<mailbox.mbox at 0x12a9cacd490>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import mailbox\n",
"\n",
"mboxfile = \"./All mail Including Spam and Trash.mbox\"\n",
"mbox = mailbox.mbox(mboxfile)\n",
"\n",
"for key in mbox[0].keys():\n",
" print(key)\n",
"mbox"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Data transformation"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Data cleansing"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import csv\n",
"\n",
"with open('testdata.csv','w') as outputfile:\n",
" writing = csv.writer(outputfile)\n",
" writing.writerow(['subject', 'from', 'date', 'to', 'label','Thread'])\n",
"\n",
" for message in mbox:\n",
" writing.writerow([\n",
" message['subject'],\n",
 " message['from'],\n",
" message['date'],\n",
" message['to'],\n",
" message['X-Gmail-Labels'],\n",
" message['X-GM-THRID']\n",
" ])\n"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Loading the csv file\n"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject</th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>to</th>\n",
" <th>label</th>\n",
" <th>Thread</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Conceptual Physics book file</td>\n",
" <td>Kadarla Sai Nithin &lt;[email protected]&gt;</td>\n",
" <td>Wed, 11 Jan 2023 10:00:04 +0530</td>\n",
" <td>Suresh Kumar Lokhande &lt;[email protected]&gt;</td>\n",
" <td>Archived,Sent</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Thank you for signing up for ThingSpeak!</td>\n",
" <td>ThingSpeak Support &lt;[email protected]&gt;</td>\n",
" <td>Sat, 22 Apr 2023 09:56:31 +0000</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Category Updates,Unread</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Get the official Gmail app</td>\n",
" <td>Gmail Team &lt;[email protected]&gt;</td>\n",
" <td>Wed, 30 Nov 2022 00:05:59 -0800</td>\n",
" <td>Kadarla Sai Nithin &lt;[email protected]&gt;</td>\n",
" <td>Inbox,Unread</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Reset Your Password</td>\n",
" <td>Xmind Notifications &lt;[email protected]....</td>\n",
" <td>Tue, 16 May 2023 15:37:20 +0000</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1766065641328425275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Security alert</td>\n",
" <td>Google &lt;[email protected]&gt;</td>\n",
" <td>Sat, 25 Feb 2023 04:51:44 GMT</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Category Updates,Unread</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>61</th>\n",
" <td>Verify Email Address</td>\n",
" <td>[email protected]</td>\n",
" <td>Sat, 22 Apr 2023 09:55:12 +0000</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1763869787228555223</td>\n",
" </tr>\n",
" <tr>\n",
" <th>62</th>\n",
" <td>uceou.edu account email verification code</td>\n",
" <td>[email protected]</td>\n",
 " <td>Sun, 19 Mar 2023 23:55:52 -0700</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1760868809297254026</td>\n",
" </tr>\n",
" <tr>\n",
" <th>63</th>\n",
" <td>uceou.edu account email verification code</td>\n",
" <td>[email protected]</td>\n",
" <td>Mon, 13 Mar 2023 23:40:25 -0700</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1760324258667829753</td>\n",
" </tr>\n",
" <tr>\n",
" <th>64</th>\n",
" <td>Security alert</td>\n",
" <td>Google &lt;[email protected]&gt;</td>\n",
" <td>Fri, 19 May 2023 10:03:42 GMT</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Opened,Category Updates</td>\n",
" <td>1766316440594475561</td>\n",
" </tr>\n",
" <tr>\n",
" <th>65</th>\n",
" <td>[Request received]</td>\n",
" <td>\"Tech (Support)\" &lt;[email protected]&gt;</td>\n",
" <td>Wed, 08 Feb 2023 14:48:12 +0000</td>\n",
" <td>\"sainithink.be25\" &lt;[email protected]&gt;</td>\n",
" <td>Inbox,Category Updates,Unread</td>\n",
" <td>1757274643381080149</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>66 rows × 6 columns</p>\n",
"</div>"
],
"text/plain": [
" subject \\\n",
"0 Conceptual Physics book file \n",
"1 Thank you for signing up for ThingSpeak! \n",
"2 Get the official Gmail app \n",
"3 Reset Your Password \n",
"4 Security alert \n",
".. ... \n",
"61 Verify Email Address \n",
"62 uceou.edu account email verification code \n",
"63 uceou.edu account email verification code \n",
"64 Security alert \n",
"65 [Request received] \n",
"\n",
" from \\\n",
"0 Kadarla Sai Nithin <[email protected]> \n",
"1 ThingSpeak Support <[email protected]> \n",
"2 Gmail Team <[email protected]> \n",
"3 Xmind Notifications <[email protected].... \n",
"4 Google <[email protected]> \n",
".. ... \n",
"61 [email protected] \n",
 "62 [email protected] \n",
"63 [email protected] \n",
"64 Google <[email protected]> \n",
"65 \"Tech (Support)\" <[email protected]> \n",
"\n",
" date \\\n",
"0 Wed, 11 Jan 2023 10:00:04 +0530 \n",
"1 Sat, 22 Apr 2023 09:56:31 +0000 \n",
"2 Wed, 30 Nov 2022 00:05:59 -0800 \n",
"3 Tue, 16 May 2023 15:37:20 +0000 \n",
"4 Sat, 25 Feb 2023 04:51:44 GMT \n",
".. ... \n",
"61 Sat, 22 Apr 2023 09:55:12 +0000 \n",
"62 Sun, 19 Mar 2023 23:55:52 -0700 \n",
"63 Mon, 13 Mar 2023 23:40:25 -0700 \n",
"64 Fri, 19 May 2023 10:03:42 GMT \n",
"65 Wed, 08 Feb 2023 14:48:12 +0000 \n",
"\n",
" to \\\n",
"0 Suresh Kumar Lokhande <[email protected]> \n",
"1 [email protected] \n",
"2 Kadarla Sai Nithin <[email protected]> \n",
"3 [email protected] \n",
"4 [email protected] \n",
".. ... \n",
"61 [email protected] \n",
"62 [email protected] \n",
"63 [email protected] \n",
"64 [email protected] \n",
"65 \"sainithink.be25\" <[email protected]> \n",
"\n",
" label Thread \n",
"0 Archived,Sent 1754524927945493361 \n",
"1 Inbox,Category Updates,Unread 1763869869345248847 \n",
"2 Inbox,Unread 1750907548775931784 \n",
"3 Inbox,Important,Opened,Category Updates 1766065641328425275 \n",
"4 Inbox,Category Updates,Unread 1758777265723926184 \n",
".. ... ... \n",
"61 Inbox,Important,Opened,Category Updates 1763869787228555223 \n",
"62 Inbox,Important,Opened,Category Updates 1760868809297254026 \n",
"63 Inbox,Important,Opened,Category Updates 1760324258667829753 \n",
"64 Inbox,Opened,Category Updates 1766316440594475561 \n",
"65 Inbox,Category Updates,Unread 1757274643381080149 \n",
"\n",
"[66 rows x 6 columns]"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dfs = pd.read_csv('testdata.csv')\n",
"dfs"
]
},
{
"attachments": {},
 "cell_type": "markdown",
"metadata": {},
"source": [
"#### Converting the data"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"subject object\n",
"from object\n",
"date object\n",
"to object\n",
"label object\n",
"Thread int64\n",
"dtype: object\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject</th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>to</th>\n",
" <th>label</th>\n",
" <th>Thread</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Conceptual Physics book file</td>\n",
" <td>Kadarla Sai Nithin &lt;[email protected]&gt;</td>\n",
" <td>2023-01-11 04:30:04+00:00</td>\n",
" <td>Suresh Kumar Lokhande &lt;[email protected]&gt;</td>\n",
" <td>Archived,Sent</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Thank you for signing up for ThingSpeak!</td>\n",
" <td>ThingSpeak Support &lt;[email protected]&gt;</td>\n",
" <td>2023-04-22 09:56:31+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Category Updates,Unread</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Get the official Gmail app</td>\n",
" <td>Gmail Team &lt;[email protected]&gt;</td>\n",
" <td>2022-11-30 08:05:59+00:00</td>\n",
" <td>Kadarla Sai Nithin &lt;[email protected]&gt;</td>\n",
" <td>Inbox,Unread</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Reset Your Password</td>\n",
" <td>Xmind Notifications &lt;[email protected]....</td>\n",
" <td>2023-05-16 15:37:20+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1766065641328425275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Security alert</td>\n",
" <td>Google &lt;[email protected]&gt;</td>\n",
" <td>2023-02-25 04:51:44+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Category Updates,Unread</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>61</th>\n",
" <td>Verify Email Address</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-04-22 09:55:12+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1763869787228555223</td>\n",
" </tr>\n",
" <tr>\n",
 " <th>62</th>\n",
" <td>uceou.edu account email verification code</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-03-20 06:55:52+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1760868809297254026</td>\n",
" </tr>\n",
" <tr>\n",
" <th>63</th>\n",
" <td>uceou.edu account email verification code</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-03-14 06:40:25+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1760324258667829753</td>\n",
" </tr>\n",
" <tr>\n",
" <th>64</th>\n",
" <td>Security alert</td>\n",
" <td>Google &lt;[email protected]&gt;</td>\n",
" <td>2023-05-19 10:03:42+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Opened,Category Updates</td>\n",
" <td>1766316440594475561</td>\n",
" </tr>\n",
" <tr>\n",
" <th>65</th>\n",
" <td>[Request received]</td>\n",
" <td>\"Tech (Support)\" &lt;[email protected]&gt;</td>\n",
" <td>2023-02-08 14:48:12+00:00</td>\n",
" <td>\"sainithink.be25\" &lt;[email protected]&gt;</td>\n",
" <td>Inbox,Category Updates,Unread</td>\n",
" <td>1757274643381080149</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>66 rows × 6 columns</p>\n",
"</div>"
],
"text/plain": [
" subject \\\n",
"0 Conceptual Physics book file \n",
"1 Thank you for signing up for ThingSpeak! \n",
"2 Get the official Gmail app \n",
"3 Reset Your Password \n",
"4 Security alert \n",
".. ... \n",
"61 Verify Email Address \n",
"62 uceou.edu account email verification code \n",
"63 uceou.edu account email verification code \n",
"64 Security alert \n",
"65 [Request received] \n",
"\n",
" from \\\n",
"0 Kadarla Sai Nithin <[email protected]> \n",
"1 ThingSpeak Support <[email protected]> \n",
"2 Gmail Team <[email protected]> \n",
"3 Xmind Notifications <[email protected].... \n",
 "4 Google <[email protected]> \n",
".. ... \n",
"61 [email protected] \n",
"62 [email protected] \n",
"63 [email protected] \n",
"64 Google <[email protected]> \n",
"65 \"Tech (Support)\" <[email protected]> \n",
"\n",
" date
to \\\n",
"0 2023-01-11 04:30:04+00:00 Suresh Kumar Lokhande
<[email protected]> \n",
"1 2023-04-22 09:56:31+00:00
[email protected] \n",
"2 2022-11-30 08:05:59+00:00 Kadarla Sai Nithin
<[email protected]> \n",
"3 2023-05-16 15:37:20+00:00
[email protected] \n",
"4 2023-02-25 04:51:44+00:00
[email protected] \n",

".. ... ... \

n",
"61 2023-04-22 09:55:12+00:00
[email protected] \n",
"62 2023-03-20 06:55:52+00:00
[email protected] \n",
"63 2023-03-14 06:40:25+00:00
[email protected] \n",
"64 2023-05-19 10:03:42+00:00
[email protected] \n",
"65 2023-02-08 14:48:12+00:00 \"sainithink.be25\"
<[email protected]> \n",
"\n",
" label Thread \n",
"0 Archived,Sent 1754524927945493361 \n",
"1 Inbox,Category Updates,Unread 1763869869345248847 \n",
"2 Inbox,Unread 1750907548775931784 \n",
"3 Inbox,Important,Opened,Category Updates 1766065641328425275 \n",
"4 Inbox,Category Updates,Unread 1758777265723926184 \n",
".. ... ... \n",
"61 Inbox,Important,Opened,Category Updates 1763869787228555223 \n",
"62 Inbox,Important,Opened,Category Updates 1760868809297254026 \n",
"63 Inbox,Important,Opened,Category Updates 1760324258667829753 \n",
"64 Inbox,Opened,Category Updates 1766316440594475561 \n",
"65 Inbox,Category Updates,Unread 1757274643381080149 \n",
"\n",
"[66 rows x 6 columns]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"print(dfs.dtypes)\n",
"dfs['date'] = dfs['date'].apply(lambda x: pd.to_datetime(x,errors='coerce',
utc=True))\n",
 "dfs"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Removing NaN values"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"dfs= dfs[dfs['date'].notna()]\n",
"dfs.to_csv(\"gmaildata.csv\")\n"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Applying descriptive statistics"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
"Int64Index: 66 entries, 0 to 65\n",
"Data columns (total 6 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 subject 66 non-null object \n",
" 1 from 66 non-null object \n",
" 2 date 66 non-null datetime64[ns, UTC]\n",
" 3 to 66 non-null object \n",
" 4 label 66 non-null object \n",
" 5 Thread 66 non-null int64 \n",
"dtypes: datetime64[ns, UTC](1), int64(1), object(4)\n",
"memory usage: 3.6+ KB\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject</th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>to</th>\n",
" <th>label</th>\n",
" <th>Thread</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Conceptual Physics book file</td>\n",
" <td>Kadarla Sai Nithin &lt;[email protected]&gt;</td>\n",
" <td>2023-01-11 04:30:04+00:00</td>\n",
" <td>Suresh Kumar Lokhande &lt;[email protected]&gt;</td>\n",
" <td>Archived,Sent</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Thank you for signing up for ThingSpeak!</td>\n",
" <td>ThingSpeak Support &lt;[email protected]&gt;</td>\n",
" <td>2023-04-22 09:56:31+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Category Updates,Unread</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Get the official Gmail app</td>\n",
" <td>Gmail Team &lt;[email protected]&gt;</td>\n",
" <td>2022-11-30 08:05:59+00:00</td>\n",
" <td>Kadarla Sai Nithin &lt;[email protected]&gt;</td>\n",
" <td>Inbox,Unread</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Reset Your Password</td>\n",
" <td>Xmind Notifications &lt;[email protected]....</td>\n",
" <td>2023-05-16 15:37:20+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Important,Opened,Category Updates</td>\n",
" <td>1766065641328425275</td>\n",
 " </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Security alert</td>\n",
" <td>Google &lt;[email protected]&gt;</td>\n",
" <td>2023-02-25 04:51:44+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>Inbox,Category Updates,Unread</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject \\\n",
"0 Conceptual Physics book file \n",
"1 Thank you for signing up for ThingSpeak! \n",
"2 Get the official Gmail app \n",
"3 Reset Your Password \n",
"4 Security alert \n",
"\n",
" from \\\n",
"0 Kadarla Sai Nithin <[email protected]> \n",
"1 ThingSpeak Support <[email protected]> \n",
"2 Gmail Team <[email protected]> \n",
"3 Xmind Notifications <[email protected].... \n",
"4 Google <[email protected]> \n",
"\n",
" date to
\\\n",
"0 2023-01-11 04:30:04+00:00 Suresh Kumar Lokhande <[email protected]>
\n",
"1 2023-04-22 09:56:31+00:00 [email protected]
\n",
"2 2022-11-30 08:05:59+00:00 Kadarla Sai Nithin <[email protected]>
\n",
"3 2023-05-16 15:37:20+00:00 [email protected]
\n",
"4 2023-02-25 04:51:44+00:00 [email protected]
\n",
"\n",
" label Thread \n",
"0 Archived,Sent 1754524927945493361 \n",
"1 Inbox,Category Updates,Unread 1763869869345248847 \n",
"2 Inbox,Unread 1750907548775931784 \n",
"3 Inbox,Important,Opened,Category Updates 1766065641328425275 \n",
"4 Inbox,Category Updates,Unread 1758777265723926184 "
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dfs.info()\n",
"dfs.head()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"In the above output 'from' field has name and email at one column\n",
"Let's refactor it"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Data Refactoring"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject</th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>to</th>\n",
" <th>label</th>\n",
" <th>Thread</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Conceptual Physics book file</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-01-11 04:30:04+00:00</td>\n",
" <td>Suresh Kumar Lokhande &lt;[email protected]&gt;</td>\n",
" <td>sent</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Thank you for signing up for ThingSpeak!</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-04-22 09:56:31+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>inbox</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Get the official Gmail app</td>\n",
" <td>[email protected]</td>\n",
" <td>2022-11-30 08:05:59+00:00</td>\n",
" <td>Kadarla Sai Nithin &lt;[email protected]&gt;</td>\n",
" <td>inbox</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Reset Your Password</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-05-16 15:37:20+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>inbox</td>\n",
" <td>1766065641328425275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Security alert</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-02-25 04:51:44+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>inbox</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>61</th>\n",
" <td>Verify Email Address</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-04-22 09:55:12+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>inbox</td>\n",
" <td>1763869787228555223</td>\n",
" </tr>\n",
" <tr>\n",
" <th>62</th>\n",
 " <td>uceou.edu account email verification code</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-03-20 06:55:52+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>inbox</td>\n",
" <td>1760868809297254026</td>\n",
" </tr>\n",
" <tr>\n",
" <th>63</th>\n",
" <td>uceou.edu account email verification code</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-03-14 06:40:25+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>inbox</td>\n",
" <td>1760324258667829753</td>\n",
" </tr>\n",
" <tr>\n",
" <th>64</th>\n",
" <td>Security alert</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-05-19 10:03:42+00:00</td>\n",
" <td>[email protected]</td>\n",
" <td>inbox</td>\n",
" <td>1766316440594475561</td>\n",
" </tr>\n",
" <tr>\n",
" <th>65</th>\n",
" <td>[Request received]</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-02-08 14:48:12+00:00</td>\n",
" <td>\"sainithink.be25\" &lt;[email protected]&gt;</td>\n",
" <td>inbox</td>\n",
" <td>1757274643381080149</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>66 rows × 6 columns</p>\n",
"</div>"
],
"text/plain": [
" subject \\\n",
"0 Conceptual Physics book file \n",
"1 Thank you for signing up for ThingSpeak! \n",
"2 Get the official Gmail app \n",
"3 Reset Your Password \n",
"4 Security alert \n",
".. ... \n",
"61 Verify Email Address \n",
"62 uceou.edu account email verification code \n",
"63 uceou.edu account email verification code \n",
"64 Security alert \n",
"65 [Request received] \n",
"\n",
" from date \\\
n",
"0 [email protected] 2023-01-11 04:30:04+00:00 \
n",
"1 [email protected] 2023-04-22 09:56:31+00:00 \
n",
 "2 [email protected] 2022-11-30 08:05:59+00:00 \
n",
"3 [email protected] 2023-05-16 15:37:20+00:00 \
n",
"4 [email protected] 2023-02-25 04:51:44+00:00 \
n",
".. ... ... \
n",
"61 [email protected] 2023-04-22 09:55:12+00:00 \
n",
"62 [email protected] 2023-03-20 06:55:52+00:00 \
n",
"63 [email protected] 2023-03-14 06:40:25+00:00 \
n",
"64 [email protected] 2023-05-19 10:03:42+00:00 \
n",
"65 [email protected] 2023-02-08 14:48:12+00:00 \
n",
"\n",
" to label
Thread \n",
"0 Suresh Kumar Lokhande <[email protected]> sent
1754524927945493361 \n",
"1 [email protected] inbox
1763869869345248847 \n",
"2 Kadarla Sai Nithin <[email protected]> inbox
1750907548775931784 \n",
"3 [email protected] inbox
1766065641328425275 \n",
"4 [email protected] inbox
1758777265723926184 \n",

".. ... ... ... \

n",
"61 [email protected] inbox
1763869787228555223 \n",
"62 [email protected] inbox
1760868809297254026 \n",
"63 [email protected] inbox
1760324258667829753 \n",
"64 [email protected] inbox
1766316440594475561 \n",
"65 \"sainithink.be25\" <[email protected]> inbox
1757274643381080149 \n",
"\n",
"[66 rows x 6 columns]"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import re\n",
"\n",
"def extract_email(string):\n",
" email = re.findall(r'<(.+?)>', string)\n",
" if not email:\n",
 " email = list(filter(lambda y: '@' in y, string.split()))\n",
" return email[0] if email else np.nan\n",
"\n",
"dfs['from'] = dfs['from'].apply(lambda x: extract_email(x))\n",
"\n",
"dfs\n",
"\n",
"myemail = '[email protected]'\n",
"dfs['label'] = dfs['from'].apply(lambda x: 'sent' if x==myemail else 'inbox')\
n",
"\n",
"dfs"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Dropping columns\n",
"\n",
"We remove the 'to' coulmn as it contains our email only. So, we removed it\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject</th>\n",
" <th>from</th>\n",
" <th>date</th>\n",
" <th>label</th>\n",
" <th>Thread</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
 " <th>0</th>\n",
" <td>Conceptual Physics book file</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-01-11 04:30:04+00:00</td>\n",
" <td>sent</td>\n",
" <td>1754524927945493361</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Thank you for signing up for ThingSpeak!</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-04-22 09:56:31+00:00</td>\n",
" <td>inbox</td>\n",
" <td>1763869869345248847</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Get the official Gmail app</td>\n",
" <td>[email protected]</td>\n",
" <td>2022-11-30 08:05:59+00:00</td>\n",
" <td>inbox</td>\n",
" <td>1750907548775931784</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Reset Your Password</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-05-16 15:37:20+00:00</td>\n",
" <td>inbox</td>\n",
" <td>1766065641328425275</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Security alert</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-02-25 04:51:44+00:00</td>\n",
" <td>inbox</td>\n",
" <td>1758777265723926184</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject
from \\\n",
"0 Conceptual Physics book file [email protected]
\n",
"1 Thank you for signing up for ThingSpeak! [email protected]
\n",
"2 Get the official Gmail app [email protected]
\n",
"3 Reset Your Password [email protected]
\n",
"4 Security alert [email protected]
\n",
"\n",
" date label Thread \n",
"0 2023-01-11 04:30:04+00:00 sent 1754524927945493361 \n",
 "1 2023-04-22 09:56:31+00:00 inbox 1763869869345248847 \n",
"2 2022-11-30 08:05:59+00:00 inbox 1750907548775931784 \n",
"3 2023-05-16 15:37:20+00:00 inbox 1766065641328425275 \n",
"4 2023-02-25 04:51:44+00:00 inbox 1758777265723926184 "
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dfs.drop(columns='to',inplace=True)\n",
"dfs.head()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Refactoring timezones\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"datetime64[ns, US/Eastern]\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>subject</th>\n",
" <th>from</th>\n",
 " <th>date</th>\n",
" <th>label</th>\n",
" <th>Thread</th>\n",
" <th>dayofweek</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Conceptual Physics book file</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-01-10 23:30:04-05:00</td>\n",
" <td>sent</td>\n",
" <td>1754524927945493361</td>\n",
" <td>Tuesday</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Thank you for signing up for ThingSpeak!</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-04-22 05:56:31-04:00</td>\n",
" <td>inbox</td>\n",
" <td>1763869869345248847</td>\n",
" <td>Saturday</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Get the official Gmail app</td>\n",
" <td>[email protected]</td>\n",
" <td>2022-11-30 03:05:59-05:00</td>\n",
" <td>inbox</td>\n",
" <td>1750907548775931784</td>\n",
" <td>Wednesday</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Reset Your Password</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-05-16 11:37:20-04:00</td>\n",
" <td>inbox</td>\n",
" <td>1766065641328425275</td>\n",
" <td>Tuesday</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Security alert</td>\n",
" <td>[email protected]</td>\n",
" <td>2023-02-24 23:51:44-05:00</td>\n",
" <td>inbox</td>\n",
" <td>1758777265723926184</td>\n",
" <td>Friday</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" subject
from \\\n",
 "0 Conceptual Physics book file [email protected]
\n",
"1 Thank you for signing up for ThingSpeak! [email protected]
\n",
"2 Get the official Gmail app [email protected]
\n",
"3 Reset Your Password [email protected]
\n",
"4 Security alert [email protected]
\n",
"\n",
" date label Thread dayofweek \n",
"0 2023-01-10 23:30:04-05:00 sent 1754524927945493361 Tuesday \n",
"1 2023-04-22 05:56:31-04:00 inbox 1763869869345248847 Saturday \n",
"2 2022-11-30 03:05:59-05:00 inbox 1750907548775931784 Wednesday \n",
"3 2023-05-16 11:37:20-04:00 inbox 1766065641328425275 Tuesday \n",
"4 2023-02-24 23:51:44-05:00 inbox 1758777265723926184 Friday "
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import datetime\n",
"import pytz\n",
"\n",
"def refactor_timezone(x):\n",
" est=pytz.timezone('US/Eastern')\n",
" return x.astimezone(est)\n",
"\n",
"dfs['date'] = dfs['date'].apply(lambda x: refactor_timezone(x))\n",
"\n",
"print(dfs['date'].dtype)\n",
"\n",
"dfs['dayofweek'] = dfs['date'].dt.strftime('%A')\n",
"\n",
"dfs.head()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"dfs.index = dfs['date']\n",
"#del dfs['date']"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Data Analysis\n"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Number of emails"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Wed, 30 Nov 2022 03:05 AM\n",
"Wed, 24 May 2023 06:31 PM\n",
"inbox 65\n",
"sent 1\n",
"Name: label, dtype: int64\n"
]
}
],
"source": [
"print(dfs.index.min().strftime('%a, %d %b %Y %I:%M %p'))\n",
"print(dfs.index.max().strftime('%a, %d %b %Y %I:%M %p'))\n",
"\n",
"print(dfs['label'].value_counts())"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Time of the day "
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"sent = dfs[dfs['label']==\"sent\"]\n",
"recieved = dfs[dfs['label']==\"inbox\"]"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"from matplotlib.ticker import MaxNLocator\n",
"from scipy import ndimage\n",
"import matplotlib.gridspec as gridspec\n",
 "import matplotlib.patches as mpatches"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"def plot_todo_vs_year(df, ax, color='CO', s=0.5, title=''):\n",
" ind = np.zeros(len(df), dtype='bool')\n",
" est = pytz.timezone('US/Eastern')\n",
" df[~ind].plot.scatter()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"## Data Cleaning\n"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Exploring data"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"Importing the dataset\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"data = pd.read_csv('employee.csv')"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"Printing the first 5 records"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.0</td>\n",
 " <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade
performance_score\n",
"0 Allen Smith 45.0 NaN NaN Operations G3
723\n",
"1 S Kumar NaN 16000.0 F Finance G0
520\n",
"2 Jack Morgan 32.0 35000.0 M Finance G2
674\n",
"3 Ying Chin 45.0 65000.0 F Sales G3
556\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2
711"
]
},
"execution_count": 18,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.head()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
 "Printing the last 5 records"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
 " <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade
performance_score\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2
711\n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3
649\n",
"6 James Authur 54.0 NaN F Operations G3
53\n",
"7 Josh Wills 54.0 52000.0 F Finance G3
901\n",
"8 Leo Duck 23.0 98000.0 M Sales G4
709"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.tail()"
]
},
{
 "attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"listing the columns\n"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Index(['name', 'age', 'income', 'gender', 'department', 'grade',\n",
" 'performance_score'],\n",
" dtype='object')\n"
]
}
],
"source": [
"print(data.columns)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"Printing the shape of the dataframe\n",
"\n",
"Output indicates --> 9 rows and 7 columns"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(9, 7)\n"
]
}
],
"source": [
"print(data.shape)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"To check the table schema, its columns, rows, data types, and missing values
in the\n",
"DataFrame\n",
"\n",
"There 7 columns, 9 rows(entries) and their names, datatypes of the column,
missing values, no. of datatypes present in the dataframe"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 9 entries, 0 to 8\n",
"Data columns (total 7 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 name 9 non-null object \n",
" 1 age 7 non-null float64\n",
" 2 income 7 non-null float64\n",
" 3 gender 7 non-null object \n",
" 4 department 9 non-null object \n",
" 5 grade 9 non-null object \n",
" 6 performance_score 9 non-null int64 \n",
"dtypes: float64(2), int64(1), object(4)\n",
"memory usage: 632.0+ bytes\n"
]
}
],
"source": [
"data.info()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"To check descriptive statistics of the dataframe"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>count</th>\n",
" <td>7.000000</td>\n",
" <td>7.000000</td>\n",
" <td>9.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mean</th>\n",
" <td>40.428571</td>\n",
" <td>52857.142857</td>\n",
" <td>610.666667</td>\n",
" </tr>\n",
" <tr>\n",
" <th>std</th>\n",
" <td>12.204605</td>\n",
" <td>26028.372797</td>\n",
" <td>235.671912</td>\n",
" </tr>\n",
" <tr>\n",
" <th>min</th>\n",
" <td>23.000000</td>\n",
" <td>16000.000000</td>\n",
" <td>53.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25%</th>\n",
" <td>31.000000</td>\n",
" <td>38500.000000</td>\n",
" <td>556.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50%</th>\n",
" <td>45.000000</td>\n",
" <td>52000.000000</td>\n",
" <td>674.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>75%</th>\n",
" <td>49.500000</td>\n",
" <td>63500.000000</td>\n",
" <td>711.000000</td>\n",
 " </tr>\n",
" <tr>\n",
" <th>max</th>\n",
" <td>54.000000</td>\n",
" <td>98000.000000</td>\n",
" <td>901.000000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" age income performance_score\n",
"count 7.000000 7.000000 9.000000\n",
"mean 40.428571 52857.142857 610.666667\n",
"std 12.204605 26028.372797 235.671912\n",
"min 23.000000 16000.000000 53.000000\n",
"25% 31.000000 38500.000000 556.000000\n",
"50% 45.000000 52000.000000 674.000000\n",
"75% 49.500000 63500.000000 711.000000\n",
"max 54.000000 98000.000000 901.000000"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.describe()"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"The above table indicates that range of age, average age, max age and min age"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Filtering data to weed out the noise"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Column-wise filtration"
]
},
{
"attachments": {},
"cell_type": "markdown",
 "metadata": {},
"source": [
"To filter columns, use filter() method\n"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>department</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>Operations</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>Finance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>Finance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>Sales</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
 " <td>Operations</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>Sales</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>Operations</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>Finance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>Sales</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name department\n",
"0 Allen Smith Operations\n",
"1 S Kumar Finance\n",
"2 Jack Morgan Finance\n",
"3 Ying Chin Sales\n",
"4 Dheeraj Patel Operations\n",
"5 Satyam Sharma Sales\n",
"6 James Authur Operations\n",
"7 Josh Wills Finance\n",
"8 Leo Duck Sales"
]
},
"execution_count": 24,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.filter(['name', 'department'])"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"We can also filter coulmns by slicing\n",
"\n"
]
},
{
"cell_type": "code",
 "execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0 Allen Smith\n",
"1 S Kumar\n",
"2 Jack Morgan\n",
"3 Ying Chin\n",
"4 Dheeraj Patel\n",
"5 Satyam Sharma\n",
"6 James Authur\n",
"7 Josh Wills\n",
"8 Leo Duck\n",
"Name: name, dtype: object"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data['name']"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"To filter multiple columns, we need a list whereas single column list is not
needed "
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
 " <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>department</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>Operations</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>Finance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>Finance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>Sales</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>Operations</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>Sales</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>Operations</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>Finance</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>Sales</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name department\n",
 "0 Allen Smith Operations\n",
"1 S Kumar Finance\n",
"2 Jack Morgan Finance\n",
"3 Ying Chin Sales\n",
"4 Dheeraj Patel Operations\n",
"5 Satyam Sharma Sales\n",
"6 James Authur Operations\n",
"7 Josh Wills Finance\n",
"8 Leo Duck Sales"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data[['name', 'department']]"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"We can also select a single column using python list"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
 " <th>0</th>\n",
" <td>Allen Smith</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name\n",
"0 Allen Smith\n",
"1 S Kumar\n",
"2 Jack Morgan\n",
"3 Ying Chin\n",
"4 Dheeraj Patel\n",
"5 Satyam Sharma\n",
"6 James Authur\n",
"7 Josh Wills\n",
"8 Leo Duck"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data[['name']]"
 ]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Row-wise filtration"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" We can filter by using indices, slicing and conditions\n",
"\n",
" Here, axis = 0 for specifying the indices and axis = 1 for columns"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Indices \n"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
 " <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade performance_score\
n",
"1 S Kumar NaN 16000.0 F Finance G0 520\
n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.filter([1,2,3], axis=0)"
]
},
{
 "attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Slicing "
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
 " <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade
performance_score\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2
711\n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3
649\n",
"6 James Authur 54.0 NaN F Operations G3
53"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data[-5:-2]"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Condition-based filtration\n",
"\n",
" For single value, '==' operator \n",
" syntax: DataFrame[DataFrame.column_name == 'Data']\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
 "</div>"
],
"text/plain": [
" name age income gender department grade performance_score\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556\
n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649\
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709"
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data[data.department=='Sales']"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" For multiple values, isin() method\n",
" syntax: DataFrame[DataFrame.column_name.isin(['value_1', 'value_2'])]"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
 " <td>Leo Duck</td>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade
performance_score\n",
"0 Allen Smith 45.0 NaN NaN Operations G3
723\n",
"3 Ying Chin 45.0 65000.0 F Sales G3
556\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2
711\n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3
649\n",
"6 James Authur 54.0 NaN F Operations G3
53\n",
"8 Leo Duck 23.0 98000.0 M Sales G4
709"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data[data.department.isin(['Sales', 'Operations'])]"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" For continuous variables, We can also use conditions like '<=' and '>='\
n",
" To Specifiy single condition:\n",
" Syntax: DataFrame[(DataFrame.column_name <= ''Value')]"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
 " <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade
performance_score\n",
"0 Allen Smith 45.0 NaN NaN Operations G3
723\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2
711\n",
"7 Josh Wills 54.0 52000.0 F Finance G3
901\n",
"8 Leo Duck 23.0 98000.0 M Sales G4
709"
]
},
"execution_count": 32,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data[(data.performance_score >=700)]"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" To specify multiple conditions\n",
" Syntax: DataFrame[(DataFrame.column_name <= ''Value') &
(DataFrame.column_name <= ''Value')]"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
 " </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade performance_score\
n",
"1 S Kumar NaN 16000.0 F Finance G0 520\
n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556\
n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649"
]
},
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data[(data.performance_score <=700) & (data.performance_score >500)]"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" We can also use query method where we specify boolean expressions\n",
" Syntax: DataFrame.query('boolean expression')"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
 " <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade performance_score\
n",
"1 S Kumar NaN 16000.0 F Finance G0 520\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556\
n",
"6 James Authur 54.0 NaN F Operations G3 53"
]
},
"execution_count": 34,
"metadata": {},
"output_type": "execute_result"
 }
],
"source": [
"data.query('performance_score < 600')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Handling missing values"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Dropping missing values\n",
" Using dropna(), we can remove the rows that containing the missing values\
n",
"\n",
" This method has 'how' argument which takes two values 'any' and 'all'\n"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
 " </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade
performance_score\n",
"2 Jack Morgan 32.0 35000.0 M Finance G2
674\n",
"3 Ying Chin 45.0 65000.0 F Sales G3
556\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2
711\n",
"7 Josh Wills 54.0 52000.0 F Finance G3
901\n",
"8 Leo Duck 23.0 98000.0 M Sales G4
709"
]
},
"execution_count": 35,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data2 = pd.read_csv('employee.csv')\n",
"\n",
"data2 = data2.dropna()\n",
"data2\n"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Filling in a missing value\n",
" \n",
" To fill a missing value, we use mean, median and mode\n",
"\n",
"\n",
" Now, Let's fill the mising values in 'age' column with mean\n"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" name age income gender department grade performance_score\
n",
"0 Allen Smith 45.0 NaN NaN Operations G3 723\
n",
"1 S Kumar NaN 16000.0 F Finance G0 520\
n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556\
n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711\
n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649\
n",
"6 James Authur 54.0 NaN F Operations G3 53\
n",
"7 Josh Wills 54.0 52000.0 F Finance G3 901\
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.000000</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>40.428571</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.000000</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.000000</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.000000</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>40.428571</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.000000</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
 " <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>23.000000</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade \\\n",
"0 Allen Smith 45.000000 NaN NaN Operations G3 \n",
"1 S Kumar 40.428571 16000.0 F Finance G0 \n",
"2 Jack Morgan 32.000000 35000.0 M Finance G2 \n",
"3 Ying Chin 45.000000 65000.0 F Sales G3 \n",
"4 Dheeraj Patel 30.000000 42000.0 F Operations G2 \n",
"5 Satyam Sharma 40.428571 62000.0 NaN Sales G3 \n",
"6 James Authur 54.000000 NaN F Operations G3 \n",
"7 Josh Wills 54.000000 52000.0 F Finance G3 \n",
"8 Leo Duck 23.000000 98000.0 M Sales G4 \n",
"\n",
" performance_score \n",
"0 723 \n",
"1 520 \n",
"2 674 \n",
"3 556 \n",
"4 711 \n",
"5 649 \n",
"6 53 \n",
"7 901 \n",
"8 709 "
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"fillingsdf = pd.read_csv('employee.csv')\n",
"print(fillingsdf)\n",
"\n",
"fillingsdf['age'] = fillingsdf.age.fillna(fillingsdf.age.mean())\n",
"fillingsdf"
 ]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Now, Let's fill the mising values in 'income' column with median"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.000000</td>\n",
" <td>52000.0</td>\n",
" <td>NaN</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>40.428571</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.000000</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.000000</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.000000</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>40.428571</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
 " <td>Josh Wills</td>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>23.000000</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade \\\n",
"0 Allen Smith 45.000000 52000.0 NaN Operations G3 \n",
"1 S Kumar 40.428571 16000.0 F Finance G0 \n",
"2 Jack Morgan 32.000000 35000.0 M Finance G2 \n",
"3 Ying Chin 45.000000 65000.0 F Sales G3 \n",
"4 Dheeraj Patel 30.000000 42000.0 F Operations G2 \n",
"5 Satyam Sharma 40.428571 62000.0 NaN Sales G3 \n",
"6 James Authur 54.000000 52000.0 F Operations G3 \n",
"7 Josh Wills 54.000000 52000.0 F Finance G3 \n",
"8 Leo Duck 23.000000 98000.0 M Sales G4 \n",
"\n",
" performance_score \n",
"0 723 \n",
"1 520 \n",
"2 674 \n",
"3 556 \n",
"4 711 \n",
"5 649 \n",
"6 53 \n",
"7 901 \n",
"8 709 "
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"fillingsdf['income'] = fillingsdf.income.fillna(fillingsdf.income.median())\
n",
"fillingsdf"
]
},
{
"attachments": {},
 "cell_type": "markdown",
"metadata": {},
"source": [
" Now, Let's fill the missing values in 'gender' column with mode"
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>40.428571</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.000000</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.000000</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.000000</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>40.428571</td>\n",
" <td>62000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>54.000000</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
 " <td>Finance</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>23.000000</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade \\\n",
"0 Allen Smith 45.000000 52000.0 F Operations G3 \n",
"1 S Kumar 40.428571 16000.0 F Finance G0 \n",
"2 Jack Morgan 32.000000 35000.0 M Finance G2 \n",
"3 Ying Chin 45.000000 65000.0 F Sales G3 \n",
"4 Dheeraj Patel 30.000000 42000.0 F Operations G2 \n",
"5 Satyam Sharma 40.428571 62000.0 F Sales G3 \n",
"6 James Authur 54.000000 52000.0 F Operations G3 \n",
"7 Josh Wills 54.000000 52000.0 F Finance G3 \n",
"8 Leo Duck 23.000000 98000.0 M Sales G4 \n",
"\n",
" performance_score \n",
"0 723 \n",
"1 520 \n",
"2 674 \n",
"3 556 \n",
"4 711 \n",
"5 649 \n",
"6 53 \n",
"7 901 \n",
"8 709 "
]
},
"execution_count": 38,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"fillingsdf['gender'] = fillingsdf['gender'].fillna(fillingsdf['gender'].mode()
[0])\n",
"fillingsdf"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
 },
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Handling outliers\n",
"\n",
" "
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Outliers are those points that are distinct from most of the similar
points"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Dropping the outliers using the standard deviation"
]
},
{
"cell_type": "code",
"execution_count": 39,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" name age income gender department grade performance_score\
n",
"0 Allen Smith 45.0 NaN NaN Operations G3 723\
n",
"1 S Kumar NaN 16000.0 F Finance G0 520\
n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556\
n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711\
n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649\
n",
"6 James Authur 54.0 NaN F Operations G3 53\
n",
"7 Josh Wills 54.0 52000.0 F Finance G3 901\
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n",
"\n",
" name age income gender department grade performance_score\
n",
"0 Allen Smith 45.0 NaN NaN Operations G3 723\
n",
"1 S Kumar NaN 16000.0 F Finance G0 520\
n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556\
n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711\
n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649\
n",
"6 James Authur 54.0 NaN F Operations G3 53\
n",
"7 Josh Wills 54.0 52000.0 F Finance G3 901\
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n"
]
}
],
"source": [
"data = pd.read_csv('employee.csv')\n",
"\n",
"# Dropping the outliers using the standard deviation \n",
"upper_limit = data['performance_score'].mean() + 3 *
data['performance_score'].std()\n",
"lower_limit = data['performance_score'].mean() - 3 *
data['performance_score'].std()\n",
"\n",
"# Upper limit = mean + 3*std\n",
"# Lower limit = mean - 3*std\n",
"\n",
"data_filtered = data[ (data['performance_score'] < upper_limit) &
(data['performance_score'] > lower_limit) ]\n",
"\n",
"print(data)\n",
"print()\n",
"print(data_filtered)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Now, Let's remove the outliers using the percentile values\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
 "text": [
" name age income gender department grade performance_score\
n",
"0 Allen Smith 45.0 NaN NaN Operations G3 723\
n",
"1 S Kumar NaN 16000.0 F Finance G0 520\
n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556\
n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711\
n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649\
n",
"6 James Authur 54.0 NaN F Operations G3 53\
n",
"7 Josh Wills 54.0 52000.0 F Finance G3 901\
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n",
"\n",
" name age income gender department grade performance_score\
n",
"0 Allen Smith 45.0 NaN NaN Operations G3 723\
n",
"1 S Kumar NaN 16000.0 F Finance G0 520\
n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674\
n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556\
n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711\
n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649\
n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709\
n"
]
}
],
"source": [
"data = pd.read_csv('employee.csv')\n",
"\n",
"# Using the percentile values to filter outliers of the data \n",
"\n",
"upper_limit = data['performance_score'].quantile(0.99)\n",
"lower_limit = data['performance_score'].quantile(0.01)\n",
"\n",
"data_filtered = data[(data['performance_score'] < upper_limit) &
(data['performance_score'] > lower_limit)]\n",
"\n",
"print(data)\n",
"print()\n",
"print(data_filtered)\n"
]
},
{
"attachments": {},
 "cell_type": "markdown",
"metadata": {},
"source": [
"### Feature encoding techniques"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### One-hot encoding\n",
" One-hot encoding transforms the categorical column into labels and splits
the column into multiple columns"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" To perform one-hot encoding, use get_dummies() method"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" <th>F</th>\n",
" <th>M</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
 " <td>711</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G3</td>\n",
" <td>901</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade performance_score
\\\n",
 "0 Allen Smith 45.0 NaN NaN Operations G3 723
\n",
"1 S Kumar NaN 16000.0 F Finance G0 520
\n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674
\n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556
\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711
\n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649
\n",
"6 James Authur 54.0 NaN F Operations G3 53
\n",
"7 Josh Wills 54.0 52000.0 F Finance G3 901
\n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709
\n",
"\n",
" F M \n",
"0 0 0 \n",
"1 1 0 \n",
"2 0 1 \n",
"3 1 0 \n",
"4 1 0 \n",
"5 0 0 \n",
"6 1 0 \n",
"7 1 0 \n",
"8 0 1 "
]
},
"execution_count": 41,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data = pd.read_csv('employee.csv')\n",
"\n",
"encoded_data = pd.get_dummies(data['gender'])\n",
"\n",
"data = data.join(encoded_data)\n",
"\n",
"data"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Previously, we have used geet_dummies() method\n",
" \n",
" Now, Let's use OneHotEncoder from scikit-learn module"
]
},
{
"cell_type": "code",
"execution_count": 42,
 "metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[[1. 0.]\n",
" [1. 0.]\n",
" [0. 1.]\n",
" [1. 0.]\n",
" [1. 0.]\n",
" [1. 0.]\n",
" [1. 0.]\n",
" [1. 0.]\n",
" [0. 1.]]\n"
]
}
],
"source": [
"from sklearn.preprocessing import OneHotEncoder\n",
"\n",
"onehotencoder = OneHotEncoder()\n",
"\n",
"data['gender'] = data['gender'].fillna(data['gender'].mode()[0])\n",
"\n",
"print(onehotencoder.fit_transform(data[['gender']]).toarray())\n",
"\n"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Label encoding\n",
" Label encoding is also known as integer encoding. Integer encoding
replaces categorical values with numeric values\n",
" "
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" Here, the unique values in variables are replaced with a sequence of
integer values"
]
},
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[1 0 0 2 1 2 1 0 2]\n"
 ]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" <th>Encoded department</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
" <td>G0</td>\n",
" <td>520</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Satyam Sharma</td>\n",
" <td>NaN</td>\n",
" <td>62000.0</td>\n",
" <td>NaN</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>649</td>\n",
" <td>2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>James Authur</td>\n",
" <td>54.0</td>\n",
" <td>NaN</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>53</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Josh Wills</td>\n",
" <td>54.0</td>\n",
" <td>52000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
 " <td>G3</td>\n",
" <td>901</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Leo Duck</td>\n",
" <td>23.0</td>\n",
" <td>98000.0</td>\n",
" <td>M</td>\n",
" <td>Sales</td>\n",
" <td>G4</td>\n",
" <td>709</td>\n",
" <td>2</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade performance_score
\\\n",
"0 Allen Smith 45.0 NaN NaN Operations G3 723
\n",
"1 S Kumar NaN 16000.0 F Finance G0 520
\n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674
\n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556
\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711
\n",
"5 Satyam Sharma NaN 62000.0 NaN Sales G3 649
\n",
"6 James Authur 54.0 NaN F Operations G3 53
\n",
"7 Josh Wills 54.0 52000.0 F Finance G3 901
\n",
"8 Leo Duck 23.0 98000.0 M Sales G4 709
\n",
"\n",
" Encoded department \n",
"0 1 \n",
"1 0 \n",
"2 0 \n",
"3 2 \n",
"4 1 \n",
"5 2 \n",
"6 1 \n",
"7 0 \n",
"8 2 "
]
},
"execution_count": 43,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
 "from sklearn.preprocessing import LabelEncoder\n",
"\n",
"data = pd.read_csv('employee.csv')\n",
"\n",
"labelencoder = LabelEncoder()\n",
"\n",
"encoded_data = labelencoder.fit_transform(data['department'])\n",
"\n",
"print(encoded_data)\n",
"\n",
"data['Encoded department'] = encoded_data\n",
"\n",
"data"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" We can also inversely transform categorical values i.e integer values to
categorical values"
]
},
{
"cell_type": "code",
"execution_count": 44,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['Finance' 'Sales' 'Operations']\n"
]
}
],
"source": [
"inverse_encoded_data = labelencoder.inverse_transform([0,2,1])\n",
"print(inverse_encoded_data)"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Ordinal encoding\n",
" It except there's an order to the encoding. The output encoding starts
from 0 to N-1"
]
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
" We can define the order of the values as a list and pass it to the
category parameter\n",
 " The ordinal encoder uses the integer or numeric values to encode where
they ordinal in nature"
]
},
{
"cell_type": "code",
"execution_count": 45,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>name</th>\n",
" <th>age</th>\n",
" <th>income</th>\n",
" <th>gender</th>\n",
" <th>department</th>\n",
" <th>grade</th>\n",
" <th>performance_score</th>\n",
" <th>grade_encoded</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Allen Smith</td>\n",
" <td>45.0</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>Operations</td>\n",
" <td>G3</td>\n",
" <td>723</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>S Kumar</td>\n",
" <td>NaN</td>\n",
" <td>16000.0</td>\n",
" <td>F</td>\n",
" <td>Finance</td>\n",
 " <td>G0</td>\n",
" <td>520</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Jack Morgan</td>\n",
" <td>32.0</td>\n",
" <td>35000.0</td>\n",
" <td>M</td>\n",
" <td>Finance</td>\n",
" <td>G2</td>\n",
" <td>674</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Ying Chin</td>\n",
" <td>45.0</td>\n",
" <td>65000.0</td>\n",
" <td>F</td>\n",
" <td>Sales</td>\n",
" <td>G3</td>\n",
" <td>556</td>\n",
" <td>3.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Dheeraj Patel</td>\n",
" <td>30.0</td>\n",
" <td>42000.0</td>\n",
" <td>F</td>\n",
" <td>Operations</td>\n",
" <td>G2</td>\n",
" <td>711</td>\n",
" <td>2.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" name age income gender department grade performance_score
\\\n",
"0 Allen Smith 45.0 NaN NaN Operations G3 723
\n",
"1 S Kumar NaN 16000.0 F Finance G0 520
\n",
"2 Jack Morgan 32.0 35000.0 M Finance G2 674
\n",
"3 Ying Chin 45.0 65000.0 F Sales G3 556
\n",
"4 Dheeraj Patel 30.0 42000.0 F Operations G2 711
\n",
"\n",
" grade_encoded \n",
"0 3.0 \n",
"1 0.0 \n",
"2 2.0 \n",
 "3 3.0 \n",
"4 2.0 "
]
},
"execution_count": 45,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from sklearn.preprocessing import OrdinalEncoder\n",
"\n",
"data = pd.read_csv('employee.csv')\n",
" \n",
"ordinalencoder = OrdinalEncoder(categories=[['G0', 'G1', 'G2', 'G3', 'G4']])\
n",
"data['grade_encoded'] = ordinalencoder.fit_transform(data[['grade']])\n",
"\n",
"data.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
"### Missing values\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
},
{
"cell_type": "code",
"execution_count": 46,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>DateTime</th>\n",
" <th>Temperature</th>\n",
" <th>Humidity</th>\n",
" <th>Wind_Speed</th>\n",
" <th>Wind_Direction</th>\n",
" <th>NO2_Location_A</th>\n",
" <th>NO2_Location_B</th>\n",
" <th>NO2_Location_C</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1/1/2020 0:00</td>\n",
" <td>2.180529</td>\n",
" <td>87</td>\n",
" <td>1.484318</td>\n",
" <td>75.963760</td>\n",
" <td>39.23</td>\n",
" <td>52.88</td>\n",
" <td>62.37</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1/1/2020 1:00</td>\n",
" <td>1.490529</td>\n",
" <td>89</td>\n",
" <td>2.741678</td>\n",
" <td>113.198590</td>\n",
" <td>38.30</td>\n",
" <td>44.95</td>\n",
" <td>52.53</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1/1/2020 2:00</td>\n",
" <td>1.690529</td>\n",
" <td>85</td>\n",
" <td>3.563818</td>\n",
" <td>135.000000</td>\n",
" <td>NaN</td>\n",
" <td>48.40</td>\n",
" <td>54.80</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1/1/2020 3:00</td>\n",
" <td>1.430529</td>\n",
" <td>84</td>\n",
" <td>2.811690</td>\n",
" <td>129.805570</td>\n",
" <td>37.28</td>\n",
" <td>47.18</td>\n",
" <td>50.80</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1/1/2020 4:00</td>\n",
" <td>0.840529</td>\n",
" <td>86</td>\n",
" <td>1.800000</td>\n",
" <td>126.869896</td>\n",
" <td>29.97</td>\n",
" <td>41.48</td>\n",
" <td>54.43</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8779</th>\n",
" <td>12/31/2020 19:00</td>\n",
" <td>4.920528</td>\n",
" <td>72</td>\n",
" <td>4.553680</td>\n",
" <td>251.565060</td>\n",
" <td>53.44</td>\n",
" <td>32.65</td>\n",
" <td>66.82</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8780</th>\n",
" <td>12/31/2020 20:00</td>\n",
" <td>4.990529</td>\n",
" <td>74</td>\n",
" <td>3.259939</td>\n",
" <td>186.340200</td>\n",
" <td>49.80</td>\n",
" <td>34.03</td>\n",
" <td>67.31</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8781</th>\n",
" <td>12/31/2020 21:00</td>\n",
" <td>4.360529</td>\n",
" <td>84</td>\n",
" <td>10.587917</td>\n",
" <td>252.181120</td>\n",
" <td>43.32</td>\n",
" <td>25.55</td>\n",
" <td>59.33</td>\n",
" </tr>\n",
 " <tr>\n",
" <th>8782</th>\n",
" <td>12/31/2020 22:00</td>\n",
" <td>3.820528</td>\n",
" <td>88</td>\n",
" <td>8.435069</td>\n",
" <td>219.805570</td>\n",
" <td>39.88</td>\n",
" <td>19.90</td>\n",
" <td>43.70</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8783</th>\n",
" <td>12/31/2020 23:00</td>\n",
" <td>3.170529</td>\n",
" <td>89</td>\n",
" <td>6.792466</td>\n",
" <td>212.005390</td>\n",
" <td>39.04</td>\n",
" <td>30.33</td>\n",
" <td>46.87</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>8784 rows × 8 columns</p>\n",
"</div>"
],
"text/plain": [
" DateTime Temperature Humidity Wind_Speed
Wind_Direction \\\n",
"0 1/1/2020 0:00 2.180529 87 1.484318 75.963760
\n",
"1 1/1/2020 1:00 1.490529 89 2.741678 113.198590
\n",
"2 1/1/2020 2:00 1.690529 85 3.563818 135.000000
\n",
"3 1/1/2020 3:00 1.430529 84 2.811690 129.805570
\n",
"4 1/1/2020 4:00 0.840529 86 1.800000 126.869896
\n",
"... ... ... ... ... ...
\n",
"8779 12/31/2020 19:00 4.920528 72 4.553680 251.565060
\n",
"8780 12/31/2020 20:00 4.990529 74 3.259939 186.340200
\n",
"8781 12/31/2020 21:00 4.360529 84 10.587917 252.181120
\n",
"8782 12/31/2020 22:00 3.820528 88 8.435069 219.805570
\n",
"8783 12/31/2020 23:00 3.170529 89 6.792466 212.005390
\n",
"\n",
" NO2_Location_A NO2_Location_B NO2_Location_C \n",
"0 39.23 52.88 62.37 \n",
"1 38.30 44.95 52.53 \n",
"2 NaN 48.40 54.80 \n",
"3 37.28 47.18 50.80 \n",
"4 29.97 41.48 54.43 \n",
 "... ... ... ... \n",
"8779 53.44 32.65 66.82 \n",
"8780 49.80 34.03 67.31 \n",
"8781 43.32 25.55 59.33 \n",
"8782 39.88 19.90 43.70 \n",
"8783 39.04 30.33 46.87 \n",
"\n",
"[8784 rows x 8 columns]"
]
},
"execution_count": 46,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"air_data = pd.read_csv('Airdata.csv')\n",
"air_data"
]
},
{
"cell_type": "code",
"execution_count": 47,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
"RangeIndex: 8784 entries, 0 to 8783\n",
"Data columns (total 8 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
" 0 DateTime 8784 non-null object \n",
" 1 Temperature 8784 non-null float64\n",
" 2 Humidity 8784 non-null int64 \n",
" 3 Wind_Speed 8784 non-null float64\n",
" 4 Wind_Direction 8784 non-null float64\n",
" 5 NO2_Location_A 8664 non-null float64\n",
" 6 NO2_Location_B 8204 non-null float64\n",
" 7 NO2_Location_C 8652 non-null float64\n",
"dtypes: float64(6), int64(1), object(1)\n",
"memory usage: 549.1+ KB\n"
]
}
],
"source": [
"air_data.info()"
]
},
{
"cell_type": "code",
"execution_count": 48,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
 "text": [
"missing values in the given dataset\n",
"--------------------------------\n",
"DateTime: 0\n",
"Temperature: 0\n",
"Humidity: 0\n",
"Wind_Speed: 0\n",
"Wind_Direction: 0\n",
"NO2_Location_A: 120\n",
"NO2_Location_B: 580\n",
"NO2_Location_C: 132\n"
]
}
],
"source": [
"print('missing values in the given dataset\\
n--------------------------------')\n",
"for col in air_data.columns:\n",
" nvm = sum(air_data[col].isna())\n",
" print('{}: {}'.format(col,nvm))\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.0"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}