284 MBox Refresher

DESCRIPTION

@@ -20,7 +20,8 @@ Authors@R: c(
     person('Nicole', 'Hoess', role = c('ctb')),
     person('Anthony', 'Lau', role = c('ctb')),
     person('Sean', 'Sunoo', role = c('ctb')),
-    person('Ian Jaymes', 'Iwata', role= c('ctb')),
+    person('Ian Jaymes', 'Iwata', role = c('ctb')),
+    person('Dao', 'McGill', role = c('ctb')),
     person('Nicholas', 'Beydler', role = c('ctb')),
     person('Mark', 'Burgess', role = c('ctb'))
     )

NAMESPACE

@@ -5,7 +5,6 @@ export(assign_exact_identity)
 export(bipartite_graph_projection)
 export(commit_message_id_coverage)
 export(community_oslom)
-export(convert_pipermail_to_mbox)
 export(dependencies_to_sdsmj)
 export(download_bugzilla_perceval_rest_issue_comments)
 export(download_bugzilla_perceval_traditional_issue_comments)
@@ -16,7 +15,6 @@ export(download_jira_issues)
 export(download_jira_issues_by_date)
 export(download_jira_issues_by_issue_key)
 export(download_mod_mbox)
-export(download_mod_mbox_per_month)
 export(download_pipermail)
 export(dv8_clsxb_to_clsxj)
 export(dv8_clsxj_to_clsxb)
@@ -184,17 +182,21 @@ export(parse_jira_rss_xml)
 export(parse_line_metrics)
 export(parse_line_type_file)
 export(parse_mbox)
+export(parse_mbox_latest_date)
 export(parse_nvdfeed)
 export(parse_r_dependencies)
 export(parse_r_function_definition)
 export(parse_r_function_dependencies)
 export(parse_rfile_ast)
+export(process_gz_to_mbox_in_folder)
 export(query_src_text)
 export(query_src_text_class_names)
 export(query_src_text_namespace)
 export(read_temporary_file)
 export(recolor_network_by_community)
 export(refresh_jira_issues)
+export(refresh_mod_mbox)
+export(refresh_pipermail)
 export(smell_missing_links)
 export(smell_organizational_silo)
 export(smell_radio_silence)

NEWS.md

@@ -3,6 +3,11 @@ __kaiaulu 0.0.0.9700 (in development)__
 
 ### NEW FEATURES
 
+ * Refactor of all R/mail.R mailing list functions for downloading and refreshing both pipermail and mod mbox archives. [#284](https://github.com/sailuh/kaiaulu/issues/284)
+ * `refresh_mod_mbox()` and `refresh_pipermail()` has been added. They are both functions that downloads mbox issues that are not already downloaded up until the current year and month. [#284](https://github.com/sailuh/kaiaulu/issues/284)
+ * `parse_mbox_latest_date()` has been added. This function returns the file name of the downloaded mbox file containing the latest date for use by `download_mbox_per_month()` and `download_pipermail` to implement a refresh capability. [#284](https://github.com/sailuh/kaiaulu/issues/284)
+ * `refresh_jira_issues()` had been added. It is a wrapper function for the previous downloader and downloads only issues greater than the greatest key already downloaded.
+ * `download_jira_issues()`, `download_jira_issues_by_issue_key()`, and `download_jira_issues_by_date()` has been added. This allows for downloading of Jira issues without the use of JirAgileR [#275](https://github.com/sailuh/kaiaulu/issues/275) and specification of issue Id and created ranges. It also interacts with `parse_jira_latest_date` to implement a refresh capability.
  * The GitHUB API has been expanded to use refresh, along with other functions. `github_api_project_issue_search` has been added that makes the search/issues endpoint API calls. `github_api_project_issue_or_pr_comments_by_date` and `github_api_project_issue_by_date` have been added to download issue data and comments by date ranges. `github_parse_search_issues_refresh` has been added that parses the issue data downloaded from the search endpoint in the refresh_issues folder. `github_api_project_issue_refresh` and `github_api_project_issue_or_pr_comment_refresh` were added to download issue data or comments respectively that have not already been downloaded. `format_created_at_from_file` was added to retrieve the greatest date from a JSON file. See the Reference Docs on GitHub section for more details. [#282](https://github.com/sailuh/kaiaulu/issues/282)
  * `config.R` now contains a set of getter functions used to centralize the gathering of configuration data and these getter functions are used to refactor configuration file information gathering. For example, loading configuration file information with variable assignment is as follows `git_repo_path <- config_file[["version_control"]][["log"]]` but refactoring with a config.R getter function becomes `git_repo_path <- get_git_repo_path(config_file)`.  [#230](https://github.com/sailuh/kaiaulu/issues/230)
  * `refresh_jira_issues()` had been added. It is a wrapper function for the previous downloader and downloads only issues greater than the greatest key already downloaded. [#275](https://github.com/sailuh/kaiaulu/issues/275)
@@ -30,7 +35,9 @@ __kaiaulu 0.0.0.9700 (in development)__
 
 ### MINOR IMPROVEMENTS
 
- * Issue #275, when introducing the concept of refresh on JIRA, affected some notebooks that still relied on data in that format. This issue change either notebook or config file to conform to the new JIRA downloader [#312](https://github.com/sailuh/kaiaulu/issues/312)
+ * `parse_mbox_latest_date()` now uses the new mbox naming convention for the latest date. [#284](https://github.com/sailuh/kaiaulu/issues/284)
+ * All mailing list documentation can now be found in `download_mail.Rmd`. [#284](https://github.com/sailuh/kaiaulu/issues/284)
+ * `download_pipermail()` now downloads all the txt and txt.gz files in the accessed pipermail archive as mbox files. [#284](https://github.com/sailuh/kaiaulu/issues/284)
  * The line metrics notebook now provides further guidance on adjusting the snapshot and filtering.
  * The R File and R Function parser can now properly parse R folders which contain folders within (not following R package structure). Both `.r` and `.R` files are also now captured (previously only one of the two were specified, but R accepts both). [#235](https://github.com/sailuh/kaiaulu/issues/235)
  * Refactor GoF Notebook in Graph GoF and Text GoF Notebooks [#224](https://github.com/sailuh/kaiaulu/issues/224)

R/example.R

@@ -550,7 +550,7 @@ example_jira_issue_comments <- function(folder_path = "/tmp", folder_name) {
 example_mailing_list_two_threads <- function(folder_path = "/tmp", folder_name, file_name) {
 
   # Create folder & repo
-  folder_path <- io_make_folder(folder_path=folder_path, folder_name = folder_name)
+  folder_path <- io_make_folder(folder_path = folder_path, folder_name = folder_name)
 
   # Step 1: Create fake mbox replies and assign them to variables for easy editing
   thread_1_reply_1 <- make_mbox_reply(mailing_list="test-list",