論文タイトルのweb scraping (rvest)

library(tidyverse) # ggplotとかdplyrとか
library(rvest) # webスクレイピング用
library(XML) # rvestで必要なので

library(stringr) # 文字列操作用
library(knitr)

extract_titles <-
  function(issue_url){
    html_issue <- 
      issue_url %>%
      read_html
    
    # 雑誌名、出版年月、巻号を取得。
    journal <-
      html_issue %>%
      html_nodes("h1#productTitle") %>% # タグがh1, IDがproductTitle
      html_text
    date <-
      html_issue %>%
      html_nodes("h2.noMargin") %>% # タグがh2, クラスがnoMargin
      html_text
    issue_volume <-
      html_issue %>%
      html_nodes("p.issueAndVolume") %>% # タグがp, クラスがissueAndVolume
      html_text
    
    # タイトル一覧を抽出
    titles_issue <-
      html_issue %>%
      # タグがdiv, クラスがcitation.tocArticle以下のaタグのついた要素を抜き出す
      html_nodes("div.citation.tocArticle a") %>%
      html_text %>%
      data_frame(title = .) %>%
      # いくつか余計な要素が含まれているので、文字列の一致不一致で抽出箇所を限定する
      filter(str_detect(title, "pages")) %>% # extract titles
      filter(!str_detect(title, "Issue Information")) %>% # exclude issue info
      mutate(journal = journal,
             date = date,
             issue_volume = issue_volume)

    # 該当Issueの１号前のIssueのTOCのURLを取得する
    previous_url <-
      html_issue %>%
      html_nodes("a.previous") %>% # タグがa, クラスがprevious
      html_attr("href") %>% # href (URLリンク) 要素を抽出
      paste0("http://onlinelibrary.wiley.com/", .)
    
    return(list(titles_issue, previous_url))
  }

extract_ISSN <-
  function(ISSN, number_parse = 10){
    # get Current issue URL from ISSN
    current_url <-
      paste0("http://onlinelibrary.wiley.com/journal/10.1111/(ISSN)", ISSN, "/issues") %>%
      read_html %>%
      html_nodes("a#currentIssueLink") %>%
      html_attr("href") %>%
      paste0("http://onlinelibrary.wiley.com/", .)

    lapply(1:number_parse, function(i){
      extract_titles(current_url) %>%
        {
          current_url <<- .[[2]] # 親環境で結果を保存
          cat("ISSN", ISSN, "; finish: volume", i, "\n", sep = "")
          .[[1]] %>% return
        }
    })
  }

NP <- "1469-8137" #New Phytologist
PCE <- "1365-3040"  # Plant, Cell & Environment
PJ <- "1365-313X" # The Plant Journal
PP <- "1399-3054" # Physiologia Plantarum
PB <- "1438-8677" # Plant Biology

title_list <-
  lapply(c(NP, PCE, PJ, PP, PB), function(i) extract_ISSN(ISSN = i, number_parse = 3))

# これだとエラー
# title_list %>%
#   dplyr::bind_rows 
# 入れ子リストだから

# purrr::flatten_XXXだと一発
# {purrr}はまだしっくり来てないがつよい  

title_df <-
  title_list %>%
  flatten_df %>%
  tidyr::extract(col = title, into = c("title", "start", "end"), regex = "(.+)\\(pages(.+[0-9]+)–([0-9]+)") %>% # remove pages
  tidyr::extract(col = issue_volume, into = c("vol", "issue"), regex = "([0-9]+).+([0-9]+)") %>% # separate volume and issue
  tidyr::extract(col = date, into = c("month", "year"), regex = "([a-zA-Z]+).([0-9]+)") # separate month and year

title_df %>%
  arrange(title) %>%
  head(20) %>%
  kable