This is an R Markdown Notebook. When you execute code within the notebook, the results appear beneath the code.
Try executing this chunk by clicking the Run button within the chunk or by placing your cursor inside it and pressing Ctrl+Shift+Enter.
# Set working directory (optional; update the path as needed)
setwd("C:/Users/casti/Downloads/MASTER BRANCH/SamuelCastillo.com/github_pages/american_statistical_association_datafest_2025")
# Load the tidyverse (includes readr, dplyr, etc.)
library(tidyverse)
Warning: package 'tidyverse' was built under R version 4.4.3
Warning: package 'ggplot2' was built under R version 4.4.3
Warning: package 'tibble' was built under R version 4.4.3
Warning: package 'tidyr' was built under R version 4.4.3
Warning: package 'readr' was built under R version 4.4.3
Warning: package 'purrr' was built under R version 4.4.3
Warning: package 'dplyr' was built under R version 4.4.3
Warning: package 'stringr' was built under R version 4.4.3
Warning: package 'forcats' was built under R version 4.4.3
Warning: package 'lubridate' was built under R version 4.4.3
── Attaching core tidyverse packages ─────────────────────────────────────────────────────────────────────────────────────── tidyverse 2.0.0 ──
✔ dplyr 1.1.4 ✔ readr 2.1.5
✔ forcats 1.0.0 ✔ stringr 1.6.0
✔ ggplot2 4.0.1 ✔ tibble 3.2.1
✔ lubridate 1.9.4 ✔ tidyr 1.3.1
✔ purrr 1.0.4
── Conflicts ───────────────────────────────────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag() masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
# 1. Load the data (similar to pd.read_csv)
data <- read_csv("Major Market Occupancy Data-revised.csv")
Rows: 190 Columns: 6
── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (2): quarter, market
dbl (4): year, ending_occupancy_proportion, starting_occupancy_proportion, a...
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
leases <- read_csv("Leases.csv")
Rows: 194685 Columns: 35
── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (17): quarter, monthsigned, market, building_name, building_id, address,...
dbl (18): year, zip, leasedSF, costarID, RBA, available_space, availability_...
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
price <- read_csv("Price and Availability Data.csv")
Rows: 1680 Columns: 18
── Column specification ───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr (3): quarter, market, internal_class
dbl (15): year, RBA, available_space, availability_proportion, internal_clas...
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.
# 2. Examine the price dataset:
colnames(price) # Similar to price.columns
[1] "year" "quarter"
[3] "market" "internal_class"
[5] "RBA" "available_space"
[7] "availability_proportion" "internal_class_rent"
[9] "overall_rent" "direct_available_space"
[11] "direct_availability_proportion" "direct_internal_class_rent"
[13] "direct_overall_rent" "sublet_available_space"
[15] "sublet_availability_proportion" "sublet_internal_class_rent"
[17] "sublet_overall_rent" "leasing"
year
quarter
market
internal_class
RBA
available_space
availability_proportion
internal_class_rent
overall_rent
direct_available_space
direct_availability_proportion
direct_internal_class_rent
direct_overall_rent
sublet_available_space
sublet_availability_proportion
sublet_internal_class_rent
sublet_overall_rent
leasing
summary(price$overall_rent) # Similar to price['overall_rent'].describe()
Min. 1st Qu. Median Mean 3rd Qu. Max.
18.75 28.28 32.29 36.74 41.07 84.75
# 3. View the head of the main data
head(data) # Similar to data.head()
# 4. Basic info about the data (similar to data.info())
glimpse(data) # Shows structure, types, etc.
Rows: 190
Columns: 6
$ year <dbl> 2020, 2020, 2020, 2020, 2020, 2020, 2020…
$ quarter <chr> "Q1", "Q1", "Q1", "Q1", "Q1", "Q1", "Q1"…
$ market <chr> "Washington D.C.", "Manhattan", "Chicago…
$ ending_occupancy_proportion <dbl> 0.19, 0.08, 0.14, 0.33, 0.20, 0.09, 0.29…
$ starting_occupancy_proportion <dbl> 0.98, 0.98, 0.99, 0.99, 0.99, 0.99, 0.99…
$ avg_occupancy_proportion <dbl> 0.78571429, 0.73285714, 0.78857143, 0.83…
# 5. Inspect columns and unique values
colnames(data) # Similar to data.columns
[1] "year" "quarter"
[3] "market" "ending_occupancy_proportion"
[5] "starting_occupancy_proportion" "avg_occupancy_proportion"
year
quarter
market
ending_occupancy_proportion
starting_occupancy_proportion
avg_occupancy_proportion
unique(data$market) # Similar to data['market'].unique()
[1] "Washington D.C." "Manhattan" "Chicago"
[4] "Houston" "Philadelphia" "San Francisco"
[7] "Los Angeles" "Dallas/Ft Worth" "South Bay/San Jose"
[10] "Austin"
Washington D.C.
Manhattan
Chicago
Houston
Philadelphia
San Francisco
Los Angeles
Dallas/Ft Worth
South Bay/San Jose
Austin
# 6. Count how many times each market appears (similar to .value_counts())
data %>%
count(market, sort = TRUE)
# 7. Calculate mean of avg_occupancy_proportion by market (similar to groupby, mean, sort)
data %>%
group_by(market) %>%
summarise(mean_avg_occupancy = mean(avg_occupancy_proportion, na.rm = TRUE)) %>%
arrange(desc(mean_avg_occupancy))
# 8. Summary statistics (similar to data.describe())
summary(data)
year quarter market
Min. :2020 Length:190 Length:190
1st Qu.:2021 Class :character Class :character
Median :2022 Mode :character Mode :character
Mean :2022
3rd Qu.:2023
Max. :2024
ending_occupancy_proportion starting_occupancy_proportion
Min. :0.0800 Min. :0.0500
1st Qu.:0.2200 1st Qu.:0.2400
Median :0.3550 Median :0.3550
Mean :0.3512 Mean :0.3779
3rd Qu.:0.4700 3rd Qu.:0.4500
Max. :0.6600 Max. :0.9900
avg_occupancy_proportion
Min. :0.05231
1st Qu.:0.28423
Median :0.41038
Mean :0.40140
3rd Qu.:0.48846
Max. :0.83857
# 9. Summary of avg_occupancy_proportion only (similar to data['avg_occupancy_proportion'].describe())
summary(data$avg_occupancy_proportion)
Min. 1st Qu. Median Mean 3rd Qu. Max.
0.05231 0.28423 0.41038 0.40140 0.48846 0.83857
# 10. Check missing values across all columns (similar to isnull().sum())
missing_values <- data %>%
summarise(across(everything(), ~ sum(is.na(.))))
missing_values
# 11. Unique values of quarter (similar to data['quarter'].unique())
unique_quarters <- unique(data$quarter)
unique_quarters
[1] "Q1" "Q2" "Q3" "Q4"
Q1
Q2
Q3
Q4
# 12. Mean of starting_occupancy_proportion (similar to data['starting_occupancy_proportion'].mean())
mean_starting_occupancy <- mean(data$starting_occupancy_proportion, na.rm = TRUE)
mean_starting_occupancy
[1] 0.3778947
# Merge datasets based on common columns (e.g., year, quarter, market)
# Ensure the column names match or rename them accordingly before merging
# Convert Python dataframes to R dataframes if needed
leases <- as.data.frame(leases)
price <- as.data.frame(price)
# Perform the merge
merged_data <- data %>%
left_join(leases, by = c("year", "quarter", "market")) %>%
left_join(price, by = c("year", "quarter", "market"))
Warning in left_join(., price, by = c("year", "quarter", "market")): Detected an unexpected many-to-many relationship between `x` and `y`.
ℹ Row 173 of `x` matches multiple rows in `y`.
ℹ Row 507 of `y` matches multiple rows in `x`.
ℹ If a many-to-many relationship is expected, set `relationship = "many-to-many"` to silence this warning.
glimpse(leases)
Rows: 194,685
Columns: 35
$ year <dbl> 2018, 2018, 2018, 2018, 2018, 2018, 201…
$ quarter <chr> "Q1", "Q1", "Q1", "Q1", "Q1", "Q1", "Q1…
$ monthsigned <chr> "01", "01", "01", "01", "01", "01", "01…
$ market <chr> "Atlanta", "Atlanta", "Atlanta", "Atlan…
$ building_name <chr> "10 Glenlake North Tower", "100 City Vi…
$ building_id <chr> "Atlanta_Central Perimeter_Atlanta_10 G…
$ address <chr> "10 Glenlake Pky NE", "3330 Cumberland …
$ region <chr> "South", "South", "South", "South", "So…
$ city <chr> "Atlanta", "Atlanta", "Atlanta", "Atlan…
$ state <chr> "GA", "GA", "GA", "GA", "GA", "GA", "GA…
$ zip <dbl> 30328, 30339, 30339, 30339, 30338, 3033…
$ internal_submarket <chr> "Central Perimeter", "Northwest", "Nort…
$ internal_class <chr> "A", "A", "A", "O", "A", "A", "A", "A",…
$ leasedSF <dbl> 24736, 965, 2215, 1925, 2404, 5091, 132…
$ company_name <chr> "Capital Investment Advisors", NA, "Efc…
$ internal_industry <chr> "Financial Services and Insurance", NA,…
$ transaction_type <chr> "Expansion", "New", "New", "New", "New"…
$ internal_market_cluster <chr> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ costarID <dbl> 445509, 436994, 434890, 434720, 437562,…
$ space_type <chr> "Relet", "Relet", "Relet", "Relet", "Re…
$ CBD_suburban <chr> "Suburban", "Suburban", "Suburban", "Su…
$ RBA <dbl> 101140416, 101140416, 101140416, 658104…
$ available_space <dbl> 20239067, 20239067, 20239067, 12728989,…
$ availability_proportion <dbl> 0.2001086, 0.2001086, 0.2001086, 0.1934…
$ internal_class_rent <dbl> 27.65589, 27.65589, 27.65589, 18.56089,…
$ overall_rent <dbl> 24.34569, 24.34569, 24.34569, 24.34569,…
$ direct_available_space <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ direct_availability_proportion <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ direct_internal_class_rent <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ direct_overall_rent <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ sublet_available_space <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ sublet_availability_proportion <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ sublet_internal_class_rent <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ sublet_overall_rent <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ leasing <dbl> 1205126, 1205126, 1205126, 715742, 1205…
glimpse(price)
Rows: 1,680
Columns: 18
$ year <dbl> 2018, 2018, 2018, 2018, 2018, 2018, 201…
$ quarter <chr> "Q1", "Q1", "Q1", "Q1", "Q1", "Q1", "Q1…
$ market <chr> "Atlanta", "Atlanta", "Austin", "Austin…
$ internal_class <chr> "A", "O", "A", "O", "A", "O", "A", "O",…
$ RBA <dbl> 101140416, 65810449, 36815073, 27947525…
$ available_space <dbl> 20239067, 12728989, 4281986, 3360936, 6…
$ availability_proportion <dbl> 0.2001086, 0.1934190, 0.1163107, 0.1210…
$ internal_class_rent <dbl> 27.65589, 18.56089, 40.38471, 30.11866,…
$ overall_rent <dbl> 24.34569, 24.34569, 36.59662, 36.59662,…
$ direct_available_space <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ direct_availability_proportion <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ direct_internal_class_rent <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ direct_overall_rent <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ sublet_available_space <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ sublet_availability_proportion <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ sublet_internal_class_rent <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ sublet_overall_rent <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, NA,…
$ leasing <dbl> 1205126, 715742, 1738905, 185674, 38075…
glimpse(merged_data)
Rows: 92,358
Columns: 53
$ year <dbl> 2020, 2020, 2020, 2020, 2020, 2020, 2…
$ quarter <chr> "Q1", "Q1", "Q1", "Q1", "Q1", "Q1", "…
$ market <chr> "Washington D.C.", "Washington D.C.",…
$ ending_occupancy_proportion <dbl> 0.19, 0.19, 0.19, 0.19, 0.19, 0.19, 0…
$ starting_occupancy_proportion <dbl> 0.98, 0.98, 0.98, 0.98, 0.98, 0.98, 0…
$ avg_occupancy_proportion <dbl> 0.7857143, 0.7857143, 0.7857143, 0.78…
$ monthsigned <chr> "01", "01", "01", "01", "01", "01", "…
$ building_name <chr> "1105 15th St NW", "1225 New York Ave…
$ building_id <chr> "Washington D.C._East End_Washington_…
$ address <chr> "1101 15th St NW", "1201 New York Ave…
$ region <chr> "Northeast", "Northeast", "Northeast"…
$ city <chr> "Washington", "Washington", "Washingt…
$ state <chr> "DC", "DC", "DC", "DC", "DC", "DC", "…
$ zip <dbl> 20005, 20005, 20005, 20005, 20006, 20…
$ internal_submarket <chr> "East End", "East End", "East End", "…
$ internal_class.x <chr> "O", "A", "A", "A", "A", "A", "O", "A…
$ leasedSF <dbl> 1163, 27900, 24196, 15934, 29520, 204…
$ company_name <chr> NA, "Accenture", "Staas & Halsey", "B…
$ internal_industry <chr> NA, "Business, Professional, and Cons…
$ transaction_type <chr> "New", "Expansion", "Renewal", "New",…
$ internal_market_cluster <chr> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ costarID <dbl> 129280, 129233, 129233, 129744, 13015…
$ space_type <chr> "Relet", "Relet", "Relet", "Relet", "…
$ CBD_suburban <chr> "CBD", "CBD", "CBD", "CBD", "CBD", "C…
$ RBA.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ available_space.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ availability_proportion.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ internal_class_rent.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ overall_rent.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ direct_available_space.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ direct_availability_proportion.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ direct_internal_class_rent.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ direct_overall_rent.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ sublet_available_space.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ sublet_availability_proportion.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ sublet_internal_class_rent.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ sublet_overall_rent.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ leasing.x <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ internal_class.y <chr> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ RBA.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ available_space.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ availability_proportion.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ internal_class_rent.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ overall_rent.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ direct_available_space.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ direct_availability_proportion.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ direct_internal_class_rent.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ direct_overall_rent.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ sublet_available_space.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ sublet_availability_proportion.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ sublet_internal_class_rent.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ sublet_overall_rent.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
$ leasing.y <dbl> NA, NA, NA, NA, NA, NA, NA, NA, NA, N…
library(ggplot2)
merged_data %>%
ggplot(aes(direct_available_space.x)) +
geom_histogram()
`stat_bin()` using `bins = 30`. Pick better value `binwidth`.
Warning: Removed 16798 rows containing non-finite outside the scale range
(`stat_bin()`).