##########
# STEP 1 #
##########
path <- "http://www.nysed.gov/admin/SEDdir.txt"
cnames <- c("adminstrator", "school", "beds", "address", "city", "state",
"zip", "area", "phone", "record", "grade")
classes <- c(rep("character", 9), rep("factor", 2))
dat <- read.csv(file = url(path), header = FALSE, strip.white = TRUE, sep= ",",
na.strings= c(" ", ""),stringsAsFactors = FALSE, col.names = cnames, colClasses = classes)
dat$locations <- as.character(with(dat, paste(address, city, state, zip, sep="+")))
NAer <- function(x) which(is.na(x)) #use to locate missing data
lapply(dat, NAer) #use to locate missing data
dat <- dat[-283, ] #remove observation 283
head(dat)
###########################################################################################
# STEP 2: CONVERT THE ADDRESS TO LAT AND LONGITUDE. #
###########################################################################################
# Many geocoding websites limit you to 2500 requests per day. #
# We will utilize two geo coding websites to work through the #
# problem faster. The two sites we will be utilizing: #
# #
# Google API: #
# browseURL("http://code.google.com/apis/maps/articles/geocodestrat.html") #
# #
# University of Southern California's geocoding web services: #
# browseURL("https://webgis.usc.edu/Services/Geocode/WebService/GeocoderWebService.aspx") #
# NOTE: This site requires you sign up for an use an api.key that you must provide to #
# the csus_geocode function. #
###########################################################################################
# GOOGLE'S API GEOCODING FUNCTION #
###################################
google_geocode <- function(ADDRESS){
require(XML)
requestXML <- function(address) {
URL <- paste("http://maps.google.com/maps/api/geocode/xml?sensor=false&address=", address, sep = "")
xmlRequest <- getURL(URL)
xml <- xmlToList(xmlRequest)
return(xml)
}
A2 <- gsub("+", ",+", ADDRESS, fixed = TRUE)
A3 <- gsub(" ", "+", A2, fixed = TRUE)
x <- requestXML(A3)
y <- c(lat = as.numeric(x$result$geometry$location$lat), lng = as.numeric(x$result$geometry$location$lng))
return(y)
} #end of function
###########################################################################################
# df <- dat[sample(1:nrow(dat), 20), ] #test it with 20 first
# df <- dat[, ] #Select specific rows (chunks for geocoding)
addresses <- df$locations
latitude <- vector("numeric", length(addresses))
longitude <- vector("numeric", length(addresses))
for (n in seq(addresses)) {
x <- google_geocode(addresses[n])
latitude[n] <- x[1]
longitude[n] <- x[2]
Sys.sleep(0.1) # Suspend requests for a tenth of a second
} # end of function
df <- cbind(df, lat = latitude, lng = longitude)
###########################################################################################
# UNIVERSITY OF SOUTHERN CALIFORNIA'S GEOCODING FUNCTION #
##########################################################
csus_geocode <- function(ADDRESS, api.key = "PUT YOUR API KEY HERE"){
require(XML)
ADDRESS <- unlist(strsplit(ADDRESS, "+", fixed = TRUE))
geocode_url <- function(address, city, state, zip, api.key){ #function to make urls
address <- gsub(" ", "+", address, fixed = TRUE)
city <- gsub(" ", "+", city, fixed = TRUE)
zip <- substring(zip, 1, 5)
z <- "http://webgis.usc.edu/Services/Geocode/WebService/GeocoderWebServiceHttpNonParsed_V02_96.aspx?"
x <- paste(z, "streetAddress=", address, "&city=", city, "&state=", state, "&zip=",
zip, "&census=false&format=xml&version=2.96&apiKey=", api.key, sep="")
return(x)
} #end of url creation helper function
requestXML <- function(URL) { #address to coordinates function
URL <- URL
xmlRequest <- getURL(URL)
xmlRequest <- substr(xmlRequest, 4, nchar(xmlRequest)) #removes 4 garbage characters at the begining
xml <- xmlToList(xmlRequest)
return(xml)
}#end of address to coordinates function
URL <- geocode_url(address = ADDRESS[1], city = ADDRESS[2], state = ADDRESS[3], zip =ADDRESS[4],
api.key = api.key)
x <- requestXML(URL)
y <- c(lat = as.numeric(x$OutputGeocode$Latitude), lng = as.numeric(x$OutputGeocode$Longitude))
return(y)
} #end of function
#####################################################################################################
# df <- dat[sample(1:nrow(dat), 20), ] #test it with 20 first
# df <- dat[, ] #Select specific rows (chunks for geocoding)
addresses <- df$locations
latitude <- vector("numeric", length(addresses))
longitude <- vector("numeric", length(addresses))
for (n in seq(addresses)) {
x <- csus_geocode(addresses[n])
latitude[n] <- x[1]
longitude[n] <- x[2]
Sys.sleep(0.1) # Suspend requests for a tenth of a second
} # end of function
df <- cbind(df, lat = latitude, lng = longitude)
#####################################################################################################
# GEOCODING A WAITING GAME :) #
########################################################################
# USE THE ABOVE TO CSUS_GEOCODING AND GOOGLE_GEOCODING FUNCTIONS TO #
# TAKE CHUNKS OF AROUND 2000 ROWS FOR GEOCODING OVER A SEVERAL DAY #
# PERIOD. I WOULD SUGGEST SAVING THE CHUNKS TO AN .RDATA FILE TO PIECE #
# BACK TOGATHER WHEN YOU'VE FINISHED GEOCODING. #
########################################################################
