Introduction to Pandas

🐼 🐼 🐼

import pandas as pd
import matplotlib.pyplot as plt

The pandas package gives us several useful tools for managing data.

! head world-population.csv

    LocID,Location,VarID,Variant,Time,MidPeriod,PopMale,PopFemale,PopTotal,PopDensity
    900,World,2,Medium,1950,1950.5,1266259.556,1270171.462,2536431.018,19.497
    900,World,2,Medium,1951,1951.5,1290237.638,1293796.589,2584034.227,19.863
    900,World,2,Medium,1952,1952.5,1313854.565,1317007.125,2630861.69,20.223
    900,World,2,Medium,1953,1953.5,1337452.786,1340156.275,2677609.061,20.582
    900,World,2,Medium,1954,1954.5,1361313.834,1363532.92,2724846.754,20.945
    900,World,2,Medium,1955,1955.5,1385658.299,1387361.616,2773019.915,21.316
    900,World,2,Medium,1956,1956.5,1410643.208,1411800.046,2822443.254,21.695
    900,World,2,Medium,1957,1957.5,1436364.62,1436941.438,2873306.058,22.086
    900,World,2,Medium,1958,1958.5,1462864.322,1462822.358,2925686.68,22.489

world_pop = pd.read_csv("world-population.csv")

world_pop

type(world_pop)

    pandas.core.frame.DataFrame

pd.read_csv returns a DataFrame object.

It stores the data as a dictionary of columns.

world_pop.head()

world_pop.tail()

You can get a whole column:

world_pop['PopTotal']

    0      2.536431e+06
    1      2.584034e+06
    2      2.630862e+06
    3      2.677609e+06
    4      2.724847e+06
               ...
    146    1.085811e+07
    147    1.086361e+07
    148    1.086835e+07
    149    1.087228e+07
    150    1.087539e+07
    Name: PopTotal, Length: 151, dtype: float64

You can get individual records:

world_pop.loc[1]

    LocID                 900
    Location            World
    VarID                   2
    Variant            Medium
    Time                 1951
    MidPeriod          1951.5
    PopMale       1290237.638
    PopFemale     1293796.589
    PopTotal      2584034.227
    PopDensity         19.863
    Name: 1, dtype: object

You can look at just the columns you want:

world_pop[['Time', 'PopTotal', 'PopMale', 'PopFemale']].head()

You can plot the data!

plt.plot(world_pop['Time'], world_pop['PopTotal'])

    [<matplotlib.lines.Line2D at 0x12a2c9130>]

world_pop.plot(x="Time", y="PopTotal")

    <AxesSubplot:xlabel='Time'>

world_pop.plot(x="Time", y=["PopMale","PopFemale","PopTotal"])

    <AxesSubplot:xlabel='Time'>

You can create new columns:

world_pop["PopTotalB"] = world_pop["PopTotal"] / 1e6

world_pop.head()

world_pop.plot(
    x='Time',
    y="PopTotalB",
    ylabel="Population (Billions)",
    title="Predicted Population for World"
)

    <AxesSubplot:title={'center':'Predicted Population for World'}, xlabel='Time', ylabel='Population (Billions)'>

🌎 🌍 🌏

world_pop = pd.read_csv("WPP2019_TotalPopulationBySex.csv")

world_pop.head()

world_pop["Location"].unique()

    array(['Afghanistan', 'Africa', 'African Group', 'African Union',
           'African Union: Central Africa', 'African Union: Eastern Africa',
           'African Union: Northern Africa', 'African Union: Southern Africa',
           'African Union: Western Africa',
           'African, Caribbean and Pacific (ACP) Group of States', 'Albania',
           'Algeria', 'American Samoa', 'Andean Community', 'Andorra',
           'Angola', 'Anguilla', 'Antigua and Barbuda', 'Argentina',
           'Armenia', 'Aruba', 'Asia',
           'Asia-Pacific Economic Cooperation (APEC)', 'Asia-Pacific Group',
           'Association of Southeast Asian Nations (ASEAN)', 'Australia',
           'Australia/New Zealand', 'Austria', 'Azerbaijan', 'BRIC', 'BRICS',
           'Bahamas', 'Bahrain', 'Bangladesh', 'Barbados', 'Belarus',
           'Belgium', 'Belize', 'Belt-Road Initiative (BRI)',
           'Belt-Road Initiative: Africa', 'Belt-Road Initiative: Asia',
           'Belt-Road Initiative: Europe',
           'Belt-Road Initiative: Latin America and the Caribbean',
           'Belt-Road Initiative: Pacific', 'Benin', 'Bermuda', 'Bhutan',
           'Black Sea Economic Cooperation (BSEC)',
           'Bolivarian Alliance for the Americas (ALBA)',
           'Bolivia (Plurinational State of)',
           'Bonaire, Sint Eustatius and Saba', 'Bosnia and Herzegovina',
           'Botswana', 'Brazil', 'British Virgin Islands',
           'Brunei Darussalam', 'Bulgaria', 'Burkina Faso', 'Burundi',
           'Cabo Verde', 'Cambodia', 'Cameroon', 'Canada', 'Caribbean',
           'Caribbean Community and Common Market (CARICOM)',
           'Cayman Islands', 'Central African Republic', 'Central America',
           'Central Asia', 'Central European Free Trade Agreement (CEFTA)',
           'Central and Southern Asia', 'Chad', 'Channel Islands', 'Chile',
           'China', 'China (and dependencies)', 'China, Hong Kong SAR',
           'China, Macao SAR', 'China, Taiwan Province of China', 'Colombia',
           'Commonwealth of Independent States (CIS)',
           'Commonwealth of Nations', 'Commonwealth: Africa',
           'Commonwealth: Asia', 'Commonwealth: Caribbean and Americas',
           'Commonwealth: Europe', 'Commonwealth: Pacific', 'Comoros',
           'Congo', 'Cook Islands', 'Costa Rica',
           'Countries with Access to the Sea',
           'Countries with Access to the Sea: Africa',
           'Countries with Access to the Sea: Asia',
           'Countries with Access to the Sea: Europe',
           'Countries with Access to the Sea: Latin America and the Caribbean',
           'Countries with Access to the Sea: Northern America',
           'Countries with Access to the Sea: Oceania', 'Croatia', 'Cuba',
           'Curaçao', 'Cyprus', 'Czechia', "Côte d'Ivoire",
           "Dem. People's Republic of Korea",
           'Democratic Republic of the Congo', 'Denmark',
           'Denmark (and dependencies)', 'Djibouti', 'Dominica',
           'Dominican Republic', 'ECE: North America-2', 'ECE: UNECE-52',
           'ECLAC: Latin America', 'ECLAC: The Caribbean',
           'ESCAP region: East and North-East Asia',
           'ESCAP region: North and Central Asia', 'ESCAP region: Pacific',
           'ESCAP region: South and South-West Asia',
           'ESCAP region: South-East Asia',
           'ESCAP: ADB Developing member countries (DMCs)',
           'ESCAP: ADB Group A (Concessional assistance\xa0only)',
           'ESCAP: ADB Group B\xa0(OCR blend)',
           'ESCAP: ADB Group C (Regular OCR only)', 'ESCAP: ASEAN',
           'ESCAP: Central Asia', 'ESCAP: ECO', 'ESCAP: HDI groups',
           'ESCAP: Landlocked countries (LLDCs)',
           'ESCAP: Least Developed Countries (LDCs)',
           'ESCAP: Pacific island dev. econ.', 'ESCAP: SAARC',
           'ESCAP: WB High income econ.', 'ESCAP: WB Low income econ.',
           'ESCAP: WB Lower middle income econ.',
           'ESCAP: WB Upper middle income econ.', 'ESCAP: WB income groups',
           'ESCAP: high HDI', 'ESCAP: high income', 'ESCAP: income groups',
           'ESCAP: low HDI', 'ESCAP: low income', 'ESCAP: lower middle HDI',
           'ESCAP: lower middle income',
           'ESCAP: other Asia-Pacific countries/areas',
           'ESCAP: upper middle HDI', 'ESCAP: upper middle income',
           'ESCWA: Arab countries', 'ESCWA: Arab least developed countries',
           'ESCWA: Gulf Cooperation Council countries',
           'ESCWA: Maghreb countries', 'ESCWA: Mashreq countries',
           'ESCWA: member countries', 'East African Community (EAC)',
           'Eastern Africa', 'Eastern Asia', 'Eastern Europe',
           'Eastern European Group', 'Eastern and South-Eastern Asia',
           'Economic Community of Central African States (ECCAS)',
           'Economic Community of West African States (ECOWAS)',
           'Economic Cooperation Organization (ECO)', 'Ecuador', 'Egypt',
           'El Salvador', 'Equatorial Guinea', 'Eritrea', 'Estonia',
           'Eswatini', 'Ethiopia', 'Eurasian Economic Community (Eurasec)',
           'Europe', 'Europe (48)', 'Europe and Northern America',
           'European Community (EC: 12)',
           'European Free Trade Agreement (EFTA)', 'European Union (EU: 15)',
           'European Union (EU: 28)', 'Falkland Islands (Malvinas)',
           'Faroe Islands', 'Fiji', 'Finland', 'France',
           'France (and dependencies)', 'French Guiana', 'French Polynesia',
           'Gabon', 'Gambia', 'Georgia', 'Germany', 'Ghana', 'Gibraltar',
           'Greater Arab Free Trade Area (GAFTA)', 'Greece', 'Greenland',
           'Grenada', 'Group of 77 (G77)', 'Group of Eight (G8)',
           'Group of Seven (G7)', 'Group of Twenty (G20) - member states',
           'Guadeloupe', 'Guam', 'Guatemala', 'Guinea', 'Guinea-Bissau',
           'Gulf Cooperation Council (GCC)', 'Guyana', 'Haiti',
           'High-income countries', 'Holy See', 'Honduras', 'Hungary',
           'Iceland', 'India', 'Indonesia', 'Iran (Islamic Republic of)',
           'Iraq', 'Ireland', 'Isle of Man', 'Israel', 'Italy', 'Jamaica',
           'Japan', 'Jordan', 'Kazakhstan', 'Kenya', 'Kiribati', 'Kuwait',
           'Kyrgyzstan', 'LLDC: Africa', 'LLDC: Asia', 'LLDC: Europe',
           'LLDC: Latin America', 'Land-locked Countries',
           'Land-locked Countries (Others)',
           'Land-locked Developing Countries (LLDC)',
           "Lao People's Democratic Republic",
           'Latin America and the Caribbean',
           'Latin American Integration Association (ALADI)',
           'Latin American and Caribbean Group (GRULAC)', 'Latvia',
           'League of Arab States (LAS, informal name: Arab League)',
           'Least developed countries', 'Least developed: Africa',
           'Least developed: Asia',
           'Least developed: Latin America and the Caribbean',
           'Least developed: Oceania', 'Lebanon', 'Lesotho',
           'Less developed regions',
           'Less developed regions, excluding China',
           'Less developed regions, excluding least developed countries',
           'Less developed: Africa', 'Less developed: Asia',
           'Less developed: Latin America and the Caribbean',
           'Less developed: Oceania', 'Liberia', 'Libya', 'Liechtenstein',
           'Lithuania', 'Low-income countries',
           'Lower-middle-income countries', 'Luxembourg', 'Madagascar',
           'Malawi', 'Malaysia', 'Maldives', 'Mali', 'Malta',
           'Marshall Islands', 'Martinique', 'Mauritania', 'Mauritius',
           'Mayotte', 'Melanesia', 'Mexico', 'Micronesia',
           'Micronesia (Fed. States of)', 'Middle Africa',
           'Middle-income countries', 'Monaco', 'Mongolia', 'Montenegro',
           'Montserrat', 'More developed regions', 'More developed: Asia',
           'More developed: Europe', 'More developed: Northern America',
           'More developed: Oceania', 'Morocco', 'Mozambique', 'Myanmar',
           'Namibia', 'Nauru', 'Nepal', 'Netherlands',
           'Netherlands (and dependencies)', 'New Caledonia',
           'New EU member states (joined since 2004)', 'New Zealand',
           'New Zealand (and dependencies)', 'Nicaragua', 'Niger', 'Nigeria',
           'Niue', 'No income group available',
           'Non-Self-Governing Territories',
           'North American Free Trade Agreement (NAFTA)',
           'North Atlantic Treaty Organization (NATO)', 'North Macedonia',
           'Northern Africa', 'Northern Africa and Western Asia',
           'Northern America', 'Northern Europe', 'Northern Mariana Islands',
           'Norway', 'Oceania',
           'Oceania (excluding Australia and New Zealand)', 'Oman',
           'Organisation for Economic Co-operation and Development (OECD)',
           'Organization for Security and Co-operation in Europe (OSCE)',
           'Organization of American States (OAS)',
           'Organization of Petroleum Exporting countries (OPEC)',
           'Organization of the Islamic Conference (OIC)', 'Pakistan',
           'Palau', 'Panama', 'Papua New Guinea', 'Paraguay', 'Peru',
           'Philippines', 'Poland', 'Polynesia', 'Portugal', 'Puerto Rico',
           'Qatar', 'Republic of Korea', 'Republic of Moldova', 'Romania',
           'Russian Federation', 'Rwanda', 'Réunion',
           'SIDS Atlantic, and Indian Ocean, Mediterranean and South China Sea (AIMS)',
           'SIDS Caribbean', 'SIDS Pacific', 'Saint Barthélemy',
           'Saint Helena', 'Saint Kitts and Nevis', 'Saint Lucia',
           'Saint Martin (French part)', 'Saint Pierre and Miquelon',
           'Saint Vincent and the Grenadines', 'Samoa', 'San Marino',
           'Sao Tome and Principe', 'Saudi Arabia', 'Senegal', 'Serbia',
           'Seychelles', 'Shanghai Cooperation Organization (SCO)',
           'Sierra Leone', 'Singapore', 'Sint Maarten (Dutch part)',
           'Slovakia', 'Slovenia', 'Small Island Developing States (SIDS)',
           'Solomon Islands', 'Somalia', 'South Africa', 'South America',
           'South Asian Association for Regional Cooperation (SAARC)',
           'South Sudan', 'South-Eastern Asia', 'Southern Africa',
           'Southern African Development Community (SADC)', 'Southern Asia',
           'Southern Common Market (MERCOSUR)', 'Southern Europe', 'Spain',
           'Sri Lanka', 'State of Palestine', 'Sub-Saharan Africa', 'Sudan',
           'Suriname', 'Sweden', 'Switzerland', 'Syrian Arab Republic',
           'Tajikistan', 'Thailand', 'Timor-Leste', 'Togo', 'Tokelau',
           'Tonga', 'Trinidad and Tobago', 'Tunisia', 'Turkey',
           'Turkmenistan', 'Turks and Caicos Islands', 'Tuvalu',
           'UN-ECE: member countries', 'UNFPA Regions',
           'UNFPA: Arab States (AS)', 'UNFPA: Asia and the Pacific (AP)',
           'UNFPA: East and Southern Africa (ESA)',
           'UNFPA: Eastern Europe and Central Asia (EECA)',
           'UNFPA: Latin America and the Caribbean (LAC)',
           'UNFPA: West and Central Africa (WCA)', 'UNICEF PROGRAMME REGIONS',
           'UNICEF Programme Regions: East Asia and Pacific (EAPRO)',
           'UNICEF Programme Regions: Eastern Caribbean',
           'UNICEF Programme Regions: Eastern and Southern Africa (ESARO)',
           'UNICEF Programme Regions: Europe and Central Asia (CEECIS)',
           'UNICEF Programme Regions: Latin America',
           'UNICEF Programme Regions: Latin America and Caribbean (LACRO)',
           'UNICEF Programme Regions: Middle East and North Africa (MENARO)',
           'UNICEF Programme Regions: South Asia (ROSA)',
           'UNICEF Programme Regions: West and Central Africa (WCARO)',
           'UNICEF REGIONS', 'UNICEF Regions: East Asia and Pacific',
           'UNICEF Regions: Eastern Europe and Central Asia',
           'UNICEF Regions: Eastern and Southern Africa',
           'UNICEF Regions: Europe and Central Asia',
           'UNICEF Regions: Latin America and Caribbean',
           'UNICEF Regions: Middle East and North Africa',
           'UNICEF Regions: North America', 'UNICEF Regions: South Asia',
           'UNICEF Regions: Sub-Saharan Africa',
           'UNICEF Regions: West and Central Africa',
           'UNICEF Regions: Western Europe',
           'UNITED NATIONS Regional Groups of Member States', 'Uganda',
           'Ukraine', 'United Arab Emirates', 'United Kingdom',
           'United Kingdom (and dependencies)',
           'United Nations Economic Commission for Africa (UN-ECA)',
           'United Nations Economic Commission for Latin America and the Caribbean (UN-ECLAC)',
           'United Nations Economic and Social Commission for Asia and the Pacific (UN-ESCAP) Regions',
           'United Nations Member States', 'United Republic of Tanzania',
           'United States Virgin Islands', 'United States of America',
           'United States of America (and dependencies)',
           'Upper-middle-income countries', 'Uruguay', 'Uzbekistan',
           'Vanuatu', 'Venezuela (Bolivarian Republic of)', 'Viet Nam',
           'WB region: East Asia and Pacific (excluding high income)',
           'WB region: Europe and Central Asia (excluding high income)',
           'WB region: Latin America and Caribbean (excluding high income)',
           'WB region: Middle East and North Africa (excluding high income)',
           'WB region: South Asia (excluding high income)',
           'WB region: Sub-Saharan Africa (excluding high income)',
           'WHO Regions', 'WHO: African region (AFRO)',
           'WHO: Americas (AMRO)', 'WHO: Eastern Mediterranean Region (EMRO)',
           'WHO: European Region (EURO)',
           'WHO: South-East Asia region (SEARO)',
           'WHO: Western Pacific region (WPRO)', 'Wallis and Futuna Islands',
           'West African Economic and Monetary Union (UEMOA)',
           'Western Africa', 'Western Asia', 'Western Europe',
           'Western European and Others Group (WEOG)', 'Western Sahara',
           'World', 'World Bank Regional Groups (developing only)', 'Yemen',
           'Zambia', 'Zimbabwe'], dtype=object)

place = "Costa Rica"
place_pop = world_pop[ world_pop["Location"].eq(place) & world_pop["Variant"].eq("Medium") ]
place_pop.plot(x="Time", y="PopTotal", title=place)

    <AxesSubplot:title={'center':'Costa Rica'}, xlabel='Time'>

world_pop["Location"].eq(place).head()

    0    False
    1    False
    2    False
    3    False
    4    False
    Name: Location, dtype: bool

world_pop["Location"].eq(place).sum()

    884

import random

places = random.sample(list(world_pop["Location"].unique()), 5)
for place in places:
    place_pop = world_pop[ world_pop["Location"].eq(place) & world_pop["Variant"].eq("Medium") ]
    place_pop.plot(x="Time", y="PopTotal", title=place)

🎥

movie_data = pd.read_csv("all-time-worldwide-box-office.csv")

movie_data.head()

def clean_money(entry):
    if isinstance(entry, float):
        return 0

    return int(entry.strip('$').replace(',', ''))

movie_data['Worldwide'] = movie_data['WorldwideBox Office'].apply(clean_money)
movie_data['Domestic'] = movie_data['DomesticBox Office'].apply(clean_money)
movie_data['International'] = movie_data['InternationalBox Office'].apply(clean_money)

movie_data['Worldwide'].describe()

    count    5.950000e+02
    mean     5.249021e+08
    std      3.130981e+08
    min      2.578052e+08
    25%      3.220422e+08
    50%      4.110029e+08
    75%      6.308023e+08
    max      2.845900e+09
    Name: Worldwide, dtype: float64

movie_data[ movie_data['Year'].eq(2019) ]

columns_of_interest = ["Rank","Year","Movie","DomesticBox Office","InternationalBox Office"]
movie_data[ movie_data['Year'].eq(2019) ][columns_of_interest]

internationals = movie_data[ movie_data["International"] / movie_data["Domestic"] > 5 ]
internationals[columns_of_interest]

domestics = movie_data[ movie_data["International"] / movie_data["Domestic"] < 0.7 ]
domestics[columns_of_interest]

movie_data['Year'].value_counts().sort_index().plot(
    figsize=(10,4),
    marker='.',
    title="Number of movies released each year",
    xlabel="Year",
    ylabel="Number of movies"
)

    <AxesSubplot:title={'center':'Number of movies released each year'}, xlabel='Year', ylabel='Number of movies'>

movie_data['Year'].value_counts().sort_index().plot(
    kind='bar',
    figsize=(10,4),
    title="Number of movies released each year",
    xlabel="Year",
    ylabel="Number of movies"
)

    <AxesSubplot:title={'center':'Number of movies released each year'}, xlabel='Year', ylabel='Number of movies'>

def about_star_wars(name):
    return "Star Wars" in name

star_wars = movie_data[ movie_data['Movie'].apply(about_star_wars)].sort_values(by='Year')
star_wars[columns_of_interest]

star_wars.plot(
    x='Year',
    y=['Domestic','International'],
    title='Star Wars Movie Revenue',
    marker='.'
)

    <AxesSubplot:title={'center':'Star Wars Movie Revenue'}, xlabel='Year'>