keyboard_arrow_down Importing Pandas

import pandas as pd
import numpy as np

keyboard_arrow_down Data Structures

keyboard_arrow_down Series
A one-dimensional array-like object.

# Creating a Series from a list

s = pd.Series([1, 3, 5, np.nan, 6, 8])
print(s)

0 1.0
1 3.0
2 5.0
3 NaN
4 6.0
5 8.0
dtype: float64

# Creating a Series with a custom index

s = pd.Series([1, 2, 3], index=['a', 'b', 'c'])
print(s)

a 1
b 2
c 3
dtype: int64

print(s.iloc[1])

s.loc['b']

NaN vs None

keyboard_arrow_down DataFrame
A two-dimensional, size-mutable, potentially heterogeneous tabular data structure.
 # Creating a DataFrame from a dictionary
import pandas as pd
data = {
'A': [1, 2, 3, 4],
'B': [5, 6, 7, 8],
'C': [9, 10, 11, 12]
}
df = pd.DataFrame(data)
print(df)

A B C
0 1 5 9
1 2 6 10
2 3 7 11
3 4 8 12

keyboard_arrow_down Basic Operations

Viewing Data

# Display the first few rows

print(df.head(3))

A B C
0 1 5 9
1 2 6 10
2 3 7 11

# Display the last few rows

print(df.tail(1))

A B C
3 4 8 12

# Display information about the DataFrame

print(df.info())

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 4 entries, 0 to 3
Data columns (total 3 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 A 4 non-null int64
1 B 4 non-null int64
2 C 4 non-null int64
dtypes: int64(3)
memory usage: 224.0 bytes
None

# Display summary statistics

print(df.describe())

A B C
count 4.000000 4.000000 4.000000
mean 2.500000 6.500000 10.500000
std 1.290994 1.290994 1.290994
min 1.000000 5.000000 9.000000
25% 1.750000 5.750000 9.750000
50% 2.500000 6.500000 10.500000
75% 3.250000 7.250000 11.250000
max 4.000000 8.000000 12.000000

# Display the DataFrame's index

print(df.index)

RangeIndex(start=0, stop=4, step=1)

# Display the DataFrame's columns

print(df.columns)

Index(['A', 'B', 'C'], dtype='object')

# Display the DataFrame's values

print(df.values)

[[ 1 5 9]
[ 2 6 10]
[ 3 7 11]
[ 4 8 12]]

keyboard_arrow_down Selecting Data

# Selecting a single column
print(df['A'])

0 1
1 2
2 3
3 4
Name: A, dtype: int64

# Selecting multiple columns

print(df[['A', 'B']])

A B
0 1 5
1 2 6
2 3 7
3 4 8

df

A B C

0 1 5 9

1 2 6 10

2 3 7 11

3 4 8 12

Next steps: Generate code with df

toggle_off View recommended plots New interactive sheet

# Selecting rows by index

print(df.loc[1]) # By label
print()
print(df.iloc[0]) # By position

A 2
B 6
C 10
Name: 1, dtype: int64

A 1
B 5
C 9
Name: 0, dtype: int64
 print(df)

A B C
0 1 5 9
1 2 6 10
2 3 7 11
3 4 8 12

# Selecting a subset of rows and columns

print(df.loc[1:2, ['A', 'C']])
print()
print(df.iloc[0:2, 0:2])

A C
1 2 10
2 3 11

A B
0 1 5
1 2 6

import pandas as pd
data = {
'A': [1, 2, 3, 4],
'B': [5, 6, 7, 8],
'C': [9, 10, 11, 12]
}
df = pd.DataFrame(data)
print(df)

A B C
0 1 5 9
1 2 6 10
2 3 7 11
3 4 8 12

a = df['A']>2
print(df[a])

A B C
2 3 7 11
3 4 8 12

# Boolean indexing
print(df['B'][df['A'] > 2])

2 7
3 8
Name: B, dtype: int64

print(df['A'] > 2)

0 False
1 False
2 True
3 True
Name: A, dtype: bool

keyboard_arrow_down Setting Values

# Setting a new column
df['D'] = df['A'] + df['B']
print(df)

A B C D
0 1 5 9 6
1 2 6 10 8
2 3 7 11 10
3 4 8 12 12

df['D1'] = [54, 65, 8, 56]

print(df)

A B C D D1
0 1 5 9 6 54
1 2 6 10 8 65
2 3 7 11 10 8
3 4 8 12 12 56

# Setting values by label

df.at[0, 'A'] = 10
print(df)

A B C D D1
0 10 5 9 6 54
1 2 6 10 8 65
2 3 7 11 10 8
3 4 8 12 12 56

# Setting values by position

df.iat[0, 1] = 20
print(df)

A B C D D1
0 10 20 9 6 54
1 2 6 10 8 65
2 3 7 11 10 8
3 4 8 12 12 56

# Setting values by boolean indexing

df.loc[df['A'] > 2, 'B'] = 99
print(df)

A B C D D1
0 10 99 9 6 54
1 2 6 10 8 65
2 3 99 11 10 8
3 4 99 12 12 56

keyboard_arrow_down Data Manipulation

Handling Missing Data

# Replacing missing values

df_with_nan = df.copy()

df_with_nan
 A B C D D1

0 10 99 9 6 54

1 2 6 10 8 65

2 3 99 11 10 8

3 4 99 12 12 56

Next steps: Generate code with df_with_nan

toggle_off View recommended plots New interactive sheet

df_with_nan.iloc[0, 1] = np.nan
df_with_nan

A B C D D1

0 10 NaN 9 6 54

1 2 6.0 10 8 65

2 3 99.0 11 10 8

3 4 99.0 12 12 56

Next steps: Generate code with df_with_nan

toggle_off View recommended plots New interactive sheet

print(df_with_nan.fillna(0)) # Replace with 0

A B C D D1
0 10 0.0 9 6 54
1 2 6.0 10 8 65
2 3 99.0 11 10 8
3 4 99.0 12 12 56

df_with_nan

A B C D D1

0 10 NaN 9 6 54

1 2 6.0 10 8 65

2 3 99.0 11 10 8

3 4 99.0 12 12 56

Next steps: Generate code with df_with_nan

toggle_off View recommended plots New interactive sheet

print(df_with_nan.dropna()) # Drop rows with missing values

A B C D D1
1 2 6.0 10 8 65
2 3 99.0 11 10 8
3 4 99.0 12 12 56

print(df_with_nan.isna())

A B C D D1
0 False True False False False
1 False False False False False
2 False False False False False
3 False False False False False

print(df_with_nan[df_with_nan['B'].isna()]) # Check for missing values

A B C D D1
0 10 NaN 9 6 54

Handling Duplicates

# Creating a DataFrame with duplicates

df_dup = pd.DataFrame({
'A': ['foo', 'bar', 'foo', 'bar', 'foo', 'bar'],
'B': [1, 1, 2, 2, 1, 1]
})
print(df_dup)
print()
# Dropping duplicates
df_no_dup = df_dup.drop_duplicates()
print("DataFrame without duplicates:\n", df_no_dup)

A B
0 foo 1
1 bar 1
2 foo 2
3 bar 2
4 foo 1
5 bar 1

DataFrame without duplicates:

A B
0 foo 1
1 bar 1
2 foo 2
3 bar 2

keyboard_arrow_down Operations
df

A B C D D1

0 10 99 9 6 54

1 2 6 10 8 65

2 3 99 11 10 8

3 4 99 12 12 56

Next steps: Generate code with df

toggle_off View recommended plots New interactive sheet

# Performing a column-wise operation

print(df.mean())

A 4.75
B 75.75
C 10.50
D 9.00
D1 45.75
dtype: float64

# Performing a row-wise operation

print(df.mean(axis=1))

0 35.6
1 18.2
2 26.2
 3 36.6
dtype: float64

df

A B C D D1

0 10 99 9 6 54

1 2 6 10 8 65

2 3 99 11 10 8

3 4 99 12 12 56

Next steps: Generate code with df

toggle_off View recommended plots New interactive sheet

import numpy as np
import pandas as pd
def value_difference(my_list):
my_array = np.array(my_list)
return np.max(my_array)-np.min(my_array)

# Applying functions to columns/rows

print(df)
print()
print(df.apply(np.cumsum))
print()
print(df.apply(value_difference,axis=1))
print()
print(df.apply(value_difference,axis=0))

A B C D D1
0 10 99 9 6 54
1 2 6 10 8 65
2 3 99 11 10 8
3 4 99 12 12 56

A B C D D1
0 10 99 9 6 54
1 12 105 19 14 119
2 15 204 30 24 127
3 19 303 42 36 183

0 93
1 63
2 96
3 95
dtype: int64

A 8
B 93
C 3
D 6
D1 57
dtype: int64

keyboard_arrow_down String Operations

# Creating a DataFrame with string data
df_str = pd.DataFrame({'A': ['foo', 'bar', 'baz'], 'B': ['one', 'two', 'three']})
df_str

A B

0 foo one

1 bar two

2 baz three

Next steps: Generate code with df_str

toggle_off View recommended plots New interactive sheet

# Applying string methods

print(df_str['A'].str.upper())
print()
print(df_str['B'].str.len())
print()
print(df_str['A'].str.contains('o'))

0 FOO
1 BAR
2 BAZ
Name: A, dtype: object

0 3
1 3
2 5
Name: B, dtype: int64

0 True
1 False
2 False
Name: A, dtype: bool

keyboard_arrow_down Merging and Joining

# Creating DataFrames to merge
df1 = pd.DataFrame({'key': ['A', 'B', 'C'], 'value': [1, 2, 3]})
df2 = pd.DataFrame({'key': ['B', 'C', 'D'], 'value': [4, 5, 6]})
print(df1)
print()
print(df2)

key value
0 A 1
1 B 2
2 C 3

key value
0 B 4
1 C 5
2 D 6

# Merging DataFrames
print(pd.merge(df1, df2, on='key')) # Inner join
print()
print(pd.merge(df1, df2, on='key', how='left')) # Left join
print()
print(pd.merge(df1, df2, on='key', how='outer')) # Outer join

key value_x value_y

0 B 2 4
1 C 3 5

key value_x value_y

0 A 1 NaN
1 B 2 4.0
2 C 3 5.0

key value_x value_y

0 A 1.0 NaN
 1 B 2.0 4.0
2 C 3.0 5.0
3 D NaN 6.0

# Concatenating DataFrames
print(pd.concat([df1, df2], axis=0)) # Concatenate rows
print()
print(pd.concat([df1, df2], axis=1)) # Concatenate columns

key value
0 A 1
1 B 2
2 C 3
0 B 4
1 C 5
2 D 6

key value key value

0 A 1 B 4
1 B 2 C 5
2 C 3 D 6

keyboard_arrow_down Grouping and Aggregating

# Creating a DataFrame to group
df_group = pd.DataFrame({
'key': ['A', 'B', 'A', 'B'],
'data': range(4)
})
df_group

key data

0 A 0

1 B 1

2 A 2

3 B 3

Next steps: Generate code with df_group

toggle_off View recommended plots New interactive sheet

# Grouping and applying aggregate functions

grouped = df_group.groupby('key').agg({'data': 'mean'})
grouped

data

key

A 1.0

B 2.0

Next steps: Generate code with grouped

toggle_off View recommended plots New interactive sheet

# Grouping and applying aggregate functions

grouped = df_group.groupby('key').agg({'data': ['mean', 'sum','size']})
grouped

data

mean sum size

key

A 1.0 2 2

B 2.0 4 2

Next steps: Generate code with grouped

toggle_off View recommended plots New interactive sheet

keyboard_arrow_down Advanced Operations

Pivot Tables

# Creating a DataFrame for pivot table

df_pivot = pd.DataFrame({
'A': ['foo', 'foo', 'foo', 'bar', 'bar', 'bar'],
'B': ['one', 'one', 'two', 'two', 'one', 'one'],
'C': ['small', 'large', 'large', 'small', 'small', 'large'],
'D': [1, 2, 2, 3, 3, 4]
})
df_pivot

A B C D

0 foo one small 1

1 foo one large 2

2 foo two large 2

3 bar two small 3

4 bar one small 3

5 bar one large 4

Next steps: Generate code with df_pivot

toggle_off View recommended plots New interactive sheet

# Creating a pivot table

pivot = df_pivot.pivot_table(values='D', index=['A', 'B'], columns=['C'])
print(pivot)

C large small
A B
bar one 4.0 3.0
two NaN 3.0
foo one 2.0 1.0
two 2.0 NaN

keyboard_arrow_down Plotting
import matplotlib.pyplot as plt

# Creating a DataFrame to plot

df_plot = pd.DataFrame({
'A': np.cumsum(np.random.randn(1000)),
'B': np.cumsum(np.random.randn(1000)),
'C': np.cumsum(np.random.randn(1000))
}, index=pd.date_range('2024-01-01', periods=1000))

# Plotting the DataFrame

df_plot.plot()
 plt.show()

url = 'https://gist.githubusercontent.com/DiogoRibeiro7/c6590d0cf119e87c39e31c21a9c0f3a8/raw/4a8e3da267a0c1f0d650901d8295a5153bde8b21/PlayTennis.csv'
df = pd.read_csv(url)
df

Outlook Temperature Humidity Wind Play Tennis

0 Sunny Hot High Weak No

1 Sunny Hot High Strong No

2 Overcast Hot High Weak Yes

3 Rain Mild High Weak Yes

4 Rain Cool Normal Weak Yes

5 Rain Cool Normal Strong No

6 Overcast Cool Normal Strong Yes

7 Sunny Mild High Weak No

8 Sunny Cool Normal Weak Yes

9 Rain Mild Normal Weak Yes

10 Sunny Mild Normal Strong Yes

11 Overcast Mild High Strong Yes

12 Overcast Hot Normal Weak Yes

13 Rain Mild High Strong No

Next steps: Generate code with df

toggle_off View recommended plots New interactive sheet

Advanced Data Manipulation

keyboard_arrow_down Creating DataFrames

import pandas as pd
data = {
'Name': ['Alice', 'Bob', 'Charlie', 'David'],
'Age': [24, 27, 22, 32],
'City': ['New York', 'Los Angeles', 'Chicago', 'Houston']
}

df = pd.DataFrame(data)
print("DataFrame from dictionary:\n", df)

DataFrame from dictionary:

Name Age City
0 Alice 24 New York
1 Bob 27 Los Angeles
2 Charlie 22 Chicago
3 David 32 Houston

From a CSV file

keyboard_arrow_down Basic DataFrame Operations

# Display the first few rows
print("First few rows:\n", df.head())

# Display the last few rows

print("Last few rows:\n", df.tail())

# Display information about the DataFrame

print("Info:\n", df.info())

# Display summary statistics

print("Summary statistics:\n", df.describe())

First few rows:

Name Age City
0 Alice 24 New York
1 Bob 27 Los Angeles
2 Charlie 22 Chicago
3 David 32 Houston
Last few rows:
Name Age City
0 Alice 24 New York
1 Bob 27 Los Angeles
2 Charlie 22 Chicago
3 David 32 Houston
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 4 entries, 0 to 3
Data columns (total 3 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 Name 4 non-null object
1 Age 4 non-null int64
2 City 4 non-null object
dtypes: int64(1), object(2)
memory usage: 224.0+ bytes
Info:
None
Summary statistics:
Age
count 4.000000
mean 26.250000
 std 4.349329
min 22.000000
25% 23.500000
50% 25.500000
75% 28.250000
max 32.000000

keyboard_arrow_down Selecting columns

# Selecting a single column
print("Single column:\n", df['Name'])

# Selecting multiple columns

print("Multiple columns:\n", df[['Name', 'City']])

Single column:
0 Alice
1 Bob
2 Charlie
3 David
Name: Name, dtype: object
Multiple columns:
Name City
0 Alice New York
1 Bob Los Angeles
2 Charlie Chicago
3 David Houston

keyboard_arrow_down Selecting rows

# Selecting a single row by index
print("Single row:\n", df.iloc[1])

# Selecting multiple rows by index

print("Multiple rows:\n", df.iloc[1:3])

# Selecting rows by conditions

print("Rows with age > 25:\n", df[df['Age'] > 25])

Single row:
Name Bob
Age 27
City Los Angeles
Name: 1, dtype: object
Multiple rows:
Name Age City
1 Bob 27 Los Angeles
2 Charlie 22 Chicago
Rows with age > 25:
Name Age City
1 Bob 27 Los Angeles
3 David 32 Houston

keyboard_arrow_down Data Manipulation

Adding a new column

df['Salary'] = [50000, 60000, 45000, 80000]

print("DataFrame with new column:\n", df)

DataFrame with new column:

Name Age City Salary
0 Alice 24 New York 50000
1 Bob 27 Los Angeles 60000
2 Charlie 22 Chicago 45000
3 David 32 Houston 80000

keyboard_arrow_down Updating values

# Update a single value
df.at[0, 'Age'] = 25
print("Updated DataFrame:\n", df)

# Update multiple values based on a condition

df.loc[df['City'] == 'New York', 'City'] = 'NYC'
print("DataFrame after updating values:\n", df)

Updated DataFrame:
Name Age City Salary
0 Alice 25 New York 50000
1 Bob 27 Los Angeles 60000
2 Charlie 22 Chicago 45000
3 David 32 Houston 80000
DataFrame after updating values:
Name Age City Salary
0 Alice 25 NYC 50000
1 Bob 27 Los Angeles 60000
2 Charlie 22 Chicago 45000
3 David 32 Houston 80000

keyboard_arrow_down Deleting columns and rows

# Delete a column
df = df.drop(columns=['Salary'])
print("DataFrame after deleting a column:\n", df)

# Delete a row
df = df.drop(index=1)
print("DataFrame after deleting a row:\n", df)

DataFrame after deleting a column:

Name Age City
0 Alice 25 NYC
1 Bob 27 Los Angeles
2 Charlie 22 Chicago
3 David 32 Houston
DataFrame after deleting a row:
Name Age City
0 Alice 25 NYC
2 Charlie 22 Chicago
3 David 32 Houston

keyboard_arrow_down Sorting data

# Sort by a single column
df_sorted = df.sort_values(by='Age')
print("Sorted by Age:\n", df_sorted)
 # Sort by multiple columns
df_sorted = df.sort_values(by=['City', 'Age'])
print("Sorted by City and Age:\n", df_sorted)

Sorted by Age:
Name Age City
2 Charlie 22 Chicago
0 Alice 25 NYC
3 David 32 Houston
Sorted by City and Age:
Name Age City
2 Charlie 22 Chicago
3 David 32 Houston
0 Alice 25 NYC

keyboard_arrow_down Basic Statistical Operations

# Calculate mean
mean_age = df['Age'].mean()
print("Mean Age:", mean_age)

# Calculate sum
sum_age = df['Age'].sum()
print("Sum of Ages:", sum_age)

# Calculate median
median_age = df['Age'].median()
print("Median Age:", median_age)

# Calculate standard deviation

std_age = df['Age'].std()
print("Standard Deviation of Age:", std_age)

Mean Age: 26.333333333333332

Sum of Ages: 79
Median Age: 25.0
Standard Deviation of Age: 5.131601439446884

keyboard_arrow_down Putting It All Together

import pandas as pd

# Creating a DataFrame from a dictionary

data = {
'Name': ['Alice', 'Bob', 'Charlie', 'David'],
'Age': [24, 27, 22, 32],
'City': ['New York', 'Los Angeles', 'Chicago', 'Houston']
}
df = pd.DataFrame(data)
print("DataFrame from dictionary:\n", df)

# Viewing data
print("First few rows:\n", df.head())
print("Summary statistics:\n", df.describe())

# Selecting columns and rows

print("Single column:\n", df['Name'])
print("Rows with age > 25:\n", df[df['Age'] > 25])

# Adding a new column

df['Salary'] = [50000, 60000, 45000, 80000]
print("DataFrame with new column:\n", df)

# Updating values
df.at[0, 'Age'] = 25
df.loc[df['City'] == 'New York', 'City'] = 'NYC'
print("Updated DataFrame:\n", df)

# Deleting columns and rows

df = df.drop(columns=['Salary'])
df = df.drop(index=1)
print("DataFrame after deleting a column and a row:\n", df)

# Sorting data
df_sorted = df.sort_values(by='Age')
print("Sorted by Age:\n", df_sorted)

# Basic statistical operations

mean_age = df['Age'].mean()
sum_age = df['Age'].sum()
median_age = df['Age'].median()
std_age = df['Age'].std()

print("Mean Age:", mean_age)

print("Sum of Ages:", sum_age)
print("Median Age:", median_age)
print("Standard Deviation of Age:", std_age)

DataFrame from dictionary:


Name Age City
0 Alice 24 New York
1 Bob 27 Los Angeles
2 Charlie 22 Chicago
3 David 32 Houston
First few rows:
Name Age City
0 Alice 24 New York
1 Bob 27 Los Angeles
2 Charlie 22 Chicago
3 David 32 Houston
Summary statistics:
Age
count 4.000000
mean 26.250000
std 4.349329
min 22.000000
25% 23.500000
50% 25.500000
75% 28.250000
max 32.000000
Single column:
0 Alice
1 Bob
2 Charlie
3 David
Name: Name, dtype: object
Rows with age > 25:
Name Age City
1 Bob 27 Los Angeles
3 David 32 Houston
DataFrame with new column:
Name Age City Salary
0 Alice 24 New York 50000
1 Bob 27 Los Angeles 60000
2 Charlie 22 Chicago 45000
3 David 32 Houston 80000
Updated DataFrame:
Name Age City Salary
0 Alice 25 NYC 50000
1 Bob 27 Los Angeles 60000
2 Charlie 22 Chicago 45000
 3 David 32 Houston 80000
DataFrame after deleting a column and a row: 
Name Age City
0 Alice 25 NYC
2 Charlie 22 Chicago
3 David 32 Houston
Sorted by Age:
Name Age City
2 Charlie 22 Chicago
0 Alice 25 NYC
3 David 32 Houston
Mean Age: 26.333333333333332
Sum of Ages: 79
Median Age: 25.0
Standard Deviation of Age: 5.131601439446884

keyboard_arrow_down Visualization with Pandas and Seaborn

import matplotlib.pyplot as plt
import seaborn as sns

# Creating a DataFrame for visualization

df_viz = pd.DataFrame({
'A': np.random.randn(100),
'B': np.random.randn(100),
'C': np.random.randn(100)
})

# Plotting with Pandas

df_viz.plot(kind='scatter', x='A', y='B')
plt.show()

# Plotting with Seaborn

sns.pairplot(df_viz)
plt.show()

keyboard_arrow_down Reading Different Files

pip install openpyxl # installing module to read excel files

Requirement already satisfied: openpyxl in /usr/local/lib/python3.10/dist-packages (3.1.5)

Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.10/dist-packages (from openpyxl) (2.0.0)

From CSV

file_path = r"/content/uber-raw-data-apr14.csv"
df = pd.read_csv(file_path)
print(df.head())

Date/Time Lat Lon Base

0 4/1/2014 0:11:00 40.7690 -73.9549 B02512
1 4/1/2014 0:17:00 40.7267 -74.0345 B02512
2 4/1/2014 0:21:00 40.7316 -73.9873 B02512
3 4/1/2014 0:28:00 40.7588 -73.9776 B02512
4 4/1/2014 0:33:00 40.7594 -73.9722 B02512

From Excel

# excel_file_path = r"C:\Users\Nileaysh\Downloads\BCADS17.xlsx"
file_path = r"/content/200c2e152dbe4ac7b301e8ae7a6762ce.xlsx"
df = pd.read_excel(file_path)
print(df.head())
 table column Unnamed: 2 Unnamed: 3 Unnamed: 4 Unnamed: 5 \
0 movie id NaN NaN NaN NaN
1 movie title NaN NaN NaN NaN
2 movie year NaN NaN NaN NaN
3 movie date_published NaN NaN NaN NaN
4 movie duration NaN NaN NaN NaN

Unnamed: 6 Unnamed: 7 Unnamed: 8 Unnamed: 9 Unnamed: 10 Unnamed: 11 \

0 NaN NaN NaN genre NaN NaN
1 NaN NaN NaN * movie_id NaN NaN
2 NaN NaN NaN * genre NaN NaN
3 NaN NaN NaN NaN NaN NaN
4 NaN NaN NaN NaN NaN NaN

Unnamed: 12
0 ratings
1 * movie_id
2 avg_rating
3 total_votes
4 median_rating