add new config and fix notebook to read it

R/github.R

@@ -484,9 +484,10 @@ github_api_page_last <- function(gh_response){
 #' @export
 #' @keywords internal
 github_api_iterate_pages <- function(token,gh_response,save_folder_path,prefix=NA,max_pages=NA,verbose=TRUE){
+
   page_number <- 1
+  data_exists <- TRUE
 
-  data_exists = TRUE
   # Set the max_pages to your api limit unless specified
   if(is.na(max_pages)){
     max_pages <- github_api_rate_limit(token)$remaining

conf/kaiaulu.yml

@@ -54,18 +54,37 @@ mailing_list:
 
 issue_tracker:
   jira:
-    # Obtained from the project's JIRA URL
-    domain: https://sailuh.atlassian.net
-    project_key: SAILUH
-    # Download using `download_jira_data.Rmd`
-    issues: ../../rawdata/issue_tracker/kaiaulu/issues/
-    issue_comments: ../../rawdata/issue_tracker/kaiaulu/issue_comments/
+    project_key_1:
+      # Obtained from the project's JIRA URL
+      domain: https://sailuh.atlassian.net
+      project_key: SAILUH
+      # Download using `download_jira_data.Rmd`
+      issues: ../../rawdata/kaiaulu/jira/issues/sailuh/
+      issue_comments: ../../rawdata/kaiaulu/jira/issue_comments/sailuh/
   github:
-    # Obtained from the project's GitHub URL
-    owner: sailuh
-    repo: kaiaulu
-    # Download using `download_github_comments.Rmd`
-    replies: ../../rawdata/github/kaiaulu
+    project_key_1:
+      # Obtained from the project's GitHub URL
+      owner: sailuh
+      repo: kaiaulu
+      # Download using `download_github_comments.Rmd`
+      issue_or_pr_comment: ../../rawdata/kaiaulu/github/issue_or_pr_comment/sailuh_kaiaulu/
+      issue: ../../rawdata/kaiaulu/github/issue/sailuh_kaiaulu/
+      issue_search: ../..rawdata/kaiaulu/github/issue_search/sailuh_kaiaulu/
+      pull_request: ../../kaiaulu/github/pull_request/sailuh_kaiaulu/
+      commit: ../../rawdata/kaiaulu/github/commit/sailuh_kaiaulu/
+    # project_key_2:
+      # # Obtained from the project's GitHub URL
+      # owner: ssunoo2
+      # repo: kaiaulu
+      # # Download using `download_github_comments.Rmd`
+      # issue_or_pr_comment: ../../rawdata/kaiaulu/github/issue_or_pr_comment/ssunoo2_kaiaulu/
+      # issue: ../../rawdata/kaiaulu/github/issue/ssunoo2_kaiaulu/
+      # refresh_issues: ../..rawdata/kaiaulu/github/refresh_issues/ssunoo2_kaiaulu
+      # pull_request: ../../kaiaulu/github/pull_request/ssunoo2_kaiaulu/
+      # commit: ../../rawdata/kaiaulu/github/commit/ssunoo2_kaiaulu/
+  #bugzilla:
+    #project_key: kaiaulu
+
 
 #vulnerabilities:
   # Folder path with nvd cve feeds (e.g. nvdcve-1.1-2018.json)

vignettes/download_github_comments.Rmd

@@ -45,12 +45,12 @@ To use the pipeline, you must specify the organization and project of interest,
 
 ```{r}
 conf <- yaml::read_yaml("../conf/kaiaulu.yml")
-save_path <- path.expand(conf[["issue_tracker"]][["github"]][["replies"]]) 
-save_path_issue_refresh <- paste0(save_path,"/issue_search/")
-save_path_issue <- paste0(save_path,"/issue/")
-save_path_pull_request <- paste0(save_path,"/pull_request/")
-save_path_issue_or_pr_comments <- paste0(save_path,"/issue_or_pr_comment/")
-save_path_commit <- paste0(save_path,"/commit/")
+
+save_path_issue_refresh <- conf[["issue_tracker"]][["github"]][["project_key_1"]][["issue_search"]]
+save_path_issue <- conf[["issue_tracker"]][["github"]][["project_key_1"]][["issue"]]
+save_path_pull_request <- conf[["issue_tracker"]][["github"]][["project_key_1"]][["pull_request"]]
+save_path_issue_or_pr_comments <- conf[["issue_tracker"]][["github"]][["project_key_1"]][["issue_or_pr_comment"]]
+save_path_commit <- conf[["issue_tracker"]][["github"]][["project_key_1"]][["commit"]]
 # Path you wish to save all raw data. A folder with the repo name and sub-folders will be created.
 owner <- conf[["issue_tracker"]][["github"]][["owner"]] # Has to match github organization (e.g. github.com/sailuh)
 repo <- conf[["issue_tracker"]][["github"]][["repo"]] # Has to match github repository (e.g. github.com/sailuh/perceive)