Merge pull request #35 from till-tietz/develop

Develop

Author: till-tietz

DESCRIPTION

@@ -1,7 +1,7 @@
 Type: Package
 Package: parsel
 Title: Parallel Dynamic Web-Scraping Using 'RSelenium'
-Version: 0.1.1
+Version: 0.2.0
 Authors@R: c(
     person("Till", "Tietz", email = "ttietz2014@gmail.com", role = c("cre","aut")))
 Description: A system to increase the efficiency of dynamic web-scraping with 'RSelenium'

NEWS.md

@@ -1,3 +1,11 @@
+#parsel 0.2.0
+
+* Added 'RSelenium' constructor functions. These functions are wrappers around 'RSelenium' methods that allow you to quickly and easily render safe, ready to use
+'RSelenium' scraping code to the console and paste it into your scraping functions. 
+Constructors can be piped together via `%>>%` to allow for intuitive, sequential 
+construction of scraping code. 
+
+
 # parsel 0.1.0
 
 * Added a `NEWS.md` file to track changes to the package.

R/constructors_elements.R

@@ -29,7 +29,7 @@ gen_varname <- function(input){
 #' @param using character string specifying locator scheme to use to search elements. Available schemes: "class name", "css selector", "id", "name", "link text", "partial link text", "tag name", "xpath".
 #' @param value character string specifying the search target.
 #' @param name character string specifying the object name the RSelenium "wElement" class object should be saved to.
-#' @param new_page logical indicating if clickElement() action will resullt in a change in url.
+#' @param new_page logical indicating if clickElement() action will result in a change in url.
 #' @param prev a placeholder for the output of functions being piped into click(). Defaults to NULL and should not be altered.
 #' @return a character string defining 'RSelenium' clicking instructions that can be pasted into a scraping function.
 #' @export
@@ -125,7 +125,7 @@ click <- function(using, value, name = NULL, new_page = FALSE, prev = NULL){
 #' @param name character string specifying the object name the RSelenium "wElement" class object should be saved to.If NULL a name will be generated automatically.
 #' @param text a character vector specifying the text to be typed.
 #' @param text_object a character string specifying the name of an external object holding the text to be typed. Note that the remDr$sendKeysToElement method only accepts list inputs.
-#' @param new_page logical indicating if sendKeysToElement() action will resullt in a change in url.
+#' @param new_page logical indicating if sendKeysToElement() action will result in a change in url.
 #' @param prev a placeholder for the output of functions being piped into type(). Defaults to NULL and should not be altered.
 #' @return a character string defining 'RSelenium' typing instructions that can be pasted into a scraping function.
 #' @export
@@ -271,8 +271,8 @@ type <- function(using, value, name = NULL, text, text_object, new_page = FALSE,
 #' @param using character string specifying locator scheme to use to search elements. Available schemes: "class name", "css selector", "id", "name", "link text", "partial link text", "tag name", "xpath".
 #' @param value character string specifying the search target.
 #' @param name character string specifying the object name the RSelenium "wElement" class object should be saved to. If NULL a name will be generated automatically.
+#' @param multiple logical indicating whether multiple elements should be returned. If TRUE the findElements() method will be invoked.
 #' @param prev a placeholder for the output of functions being piped into get_element(). Defaults to NULL and should not be altered.
-#' @param multiple logical indicating whether multiple elements should be returned. If TRUE the findElements() method will be invoced.
 #' @return a character string defining 'RSelenium' getElementText() instructions that can be pasted into a scraping function.
 #' @export
 #'
@@ -342,34 +342,32 @@ get_element <- function(using, value, name = NULL, multiple = FALSE, prev = NULL
   }
 
 
-  if(multiple == FALSE){
+  if(multiple){
 
-    finding <- paste(name, " <- ", "try(", "remDr$findElement(using = '", using,"', '", value, "')", ")", sep = "")
+    finding <- paste(name, " <- ", "try(", "remDr$findElements(using = '", using,"', '", value, "')", ")", sep = "")
 
     out <- paste(finding,
                  paste("if(is(", name, ",'try-error')){", sep = ""),
                  paste(name, " <- NA", sep = ""),
                  "} else {",
-                 paste(name, " <- ", name,"$getElementText()", sep = ""),
+                 paste(name, " <- ", "lapply(", name, ", function(i) ","i$getElementText())", sep = ""),
                  "}",
                  sep = " \n")
 
   } else {
 
-    finding <- paste(name, " <- ", "try(", "remDr$findElements(using = '", using,"', '", value, "')", ")", sep = "")
+    finding <- paste(name, " <- ", "try(", "remDr$findElement(using = '", using,"', '", value, "')", ")", sep = "")
 
     out <- paste(finding,
                  paste("if(is(", name, ",'try-error')){", sep = ""),
                  paste(name, " <- NA", sep = ""),
                  "} else {",
-                 paste(name, " <- ", "lapply(", name, ", function(i) ","i$getElementText())", sep = ""),
+                 paste(name, " <- ", name,"$getElementText()", sep = ""),
                  "}",
                  sep = " \n")
 
   }
 
-
-
   if(!is.null(prev)){
     out <- paste(prev, out, sep = " \n \n ")
   }

R/constructors_navigation.R

@@ -1,4 +1,4 @@
-#' wrapper around remDr$mavigate method to generate safe navigation code
+#' wrapper around remDr$navigate method to generate safe navigation code
 #'
 #' @param url a character string specifying the name of the object holding the url string or the url string the function should navigate to.
 #' @param prev a placeholder for the output of functions being piped into go(). Defaults to NULL and should not be altered.
@@ -82,3 +82,43 @@ goback <- function(prev = NULL){
   return(out)
 }
 
+
+#' wrapper around remDr$goForward method to generate safe forwards navigation code
+#'
+#' @param prev a placeholder for the output of functions being piped into goforward(). Defaults to NULL and should not be altered.
+#' @return a character string defining 'RSelenium' forward navigation instructions that can be pasted into a scraping function.
+#' @export
+#'
+#' @examples
+#' \dontrun{
+#'
+#' goforward() %>>%
+#' show()
+#'
+#' }
+
+goforward <- function(prev = NULL){
+
+  not_forward <- "not_forward <- TRUE"
+
+  from <- "from <- seleniumPipes::getCurrentUrl(remDr)"
+
+  go_forward <- "remDr$goForward()"
+
+  while_loop <- paste("while(not_forward){",
+                      "Sys.sleep(0.25)",
+                      "current <- seleniumPipes::getCurrentUrl(remDr)",
+                      "if(current != from){",
+                      "not_forward <- FALSE",
+                      "}",
+                      "}",
+                      sep = "\n")
+
+  out <- paste("# navigate forward to new url", not_forward, from, go_forward, while_loop, sep = "\n")
+
+  if(!is.null(prev)){
+    out <- paste(prev, out, sep = " \n \n ")
+  }
+
+  return(out)
+}

R/constructors_pipe.R

@@ -1,7 +1,7 @@
-#' pipe-like operator that passes the ouput of lhs to the prev argument of rhs to paste together a scraper function in sequence.
+#' pipe-like operator that passes the output of lhs to the prev argument of rhs to paste together a scraper function in sequence.
 #'
 #' @param lhs a parsel constructor function call
-#' @param rhs a parsel constructor fuction call that should accept lhs as its prev argument
+#' @param rhs a parsel constructor function call that should accept lhs as its prev argument
 #' @return the output of rhs evaluated with lhs as the prev argument
 #' @export
 #'

R/parscrape.R

@@ -39,7 +39,16 @@
 #'}
 
 
-parscrape <- function(scrape_fun, scrape_input, cores = NULL, packages = c("base"), browser, ports = NULL, chunk_size = NULL, scrape_tries = 1, proxy = NULL, extraCapabilities = list()) {
+parscrape <- function(scrape_fun,
+                      scrape_input,
+                      cores = NULL,
+                      packages = c("base"),
+                      browser,
+                      ports = NULL,
+                      chunk_size = NULL,
+                      scrape_tries = 1,
+                      proxy = NULL,
+                      extraCapabilities = list()) {
 
 
   if(!is.function(scrape_fun)){

cran-comments.md

@@ -1,11 +1,4 @@
-## Resubmission 
-This is a resubmission. In this version I have: 
-
-* written all package names in single quotes. Changed RSelenium to 'RSelenium' in Description title. 
-* added \value tag to close_rselenium.Rd
-* converted close_rselenium to internal function
-* added executable example to parscrape.Rd
-* ensured that examples and tests use at most 2 cores 
+This is a minor release introducing new user facing functionality. 
 
 ## Test environments
 * local Windows install, R 4.1.1
@@ -18,6 +11,6 @@ This is a resubmission. In this version I have:
 
 ## R CMD check results
 
-0 errors | 0 warnings | 1 note
+0 errors | 0 warnings | 0 note
+
 
-* This is a new release.