From dc6a001922a088ff486d0854e6db6bf5cb9964a4 Mon Sep 17 00:00:00 2001 From: henrykironde Date: Wed, 27 Feb 2019 17:32:18 -0500 Subject: [PATCH 1/4] Test retriever version --- .travis.yml | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 24bbab6..924542a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,16 +12,41 @@ addons: - libgsl0-dbg - libgsl0-dev - libgsl0ldbl - - "python3" - - "python3-pip" + - python3-dev + - python3 update: true before_install: + - export PATH=$(which python$R_PYTHON_VERSION):$PATH + - export PYTHON=$(which python$R_PYTHON_VERSION) + - export PYTHONPATH=$(which python$R_PYTHON_VERSION):$PYTHONPATH +install: - sudo apt-get update -qq - sudo apt-get install libudunits2-dev - - sudo apt-get install python3 - - sudo pip3 install --upgrade pip - - sudo pip3 install git+https://git@github.com/weecology/retriever.git --user + + # install Retriever python library + - git clone https://github.com/weecology/retriever.git retriever + - cd retriever + - wget https://bootstrap.pypa.io/get-pip.py + - sudo $PYTHON get-pip.py + - rm -rf get-pip.py + - sudo $PYTHON -m pip install -r requirements.txt + - sudo $PYTHON -m pip install pytest-cov -U + - sudo $PYTHON -m pip install pytest-xdist -U + - sudo $PYTHON -m pip install . -U + - cd .. + - rm -rf retriever/ + - sudo rm -rf ~/.retriever + + # Exports + - export PYTHON_CONFIG=python${R_PYTHON_VERSION}-config + - export PKG_LIBS=`$PYTHON_CONFIG --ldflags` + - export PKG_CPPFLAGS=`$PYTHON_CONFIG --includes` + - export PKG_CPPFLAGS+=" -D PYTHONLIBFILE=\\\"libpython${R_PYTHON_VERSION}.so\\\"" + +scritp: + - retriever ls + - retriever --version matrix: include: From b2eb4e085f147e767cea713efd853920090b2867 Mon Sep 17 00:00:00 2001 From: henrykironde Date: Wed, 27 Feb 2019 20:05:27 -0500 Subject: [PATCH 2/4] Use Docker --- .travis.yml | 81 +++++++++++++---------------------------- Dockerfile | 63 ++++++++++++++++++++++++++++++++ cli_tools/.my.cnf | 5 +++ cli_tools/.pgpass | 1 + cli_tools/entrypoint.sh | 8 ++++ load_and_test.R | 17 +++++++++ 6 files changed, 120 insertions(+), 55 deletions(-) create mode 100644 Dockerfile create mode 100644 cli_tools/.my.cnf create mode 100644 cli_tools/.pgpass create mode 100644 cli_tools/entrypoint.sh create mode 100644 load_and_test.R diff --git a/.travis.yml b/.travis.yml index 924542a..2744a1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,66 +1,37 @@ -language: r cache: - packages - pip sudo: required dist: "trusty" warnings_are_errors: false - -addons: - apt: - packages: - - libgsl0-dbg - - libgsl0-dev - - libgsl0ldbl - - python3-dev - - python3 - update: true +services: + - docker before_install: - - export PATH=$(which python$R_PYTHON_VERSION):$PATH - - export PYTHON=$(which python$R_PYTHON_VERSION) - - export PYTHONPATH=$(which python$R_PYTHON_VERSION):$PYTHONPATH -install: - - sudo apt-get update -qq - - sudo apt-get install libudunits2-dev - - # install Retriever python library - - git clone https://github.com/weecology/retriever.git retriever - - cd retriever - - wget https://bootstrap.pypa.io/get-pip.py - - sudo $PYTHON get-pip.py - - rm -rf get-pip.py - - sudo $PYTHON -m pip install -r requirements.txt - - sudo $PYTHON -m pip install pytest-cov -U - - sudo $PYTHON -m pip install pytest-xdist -U - - sudo $PYTHON -m pip install . -U - - cd .. - - rm -rf retriever/ - - sudo rm -rf ~/.retriever +# Set up environment and create required images +# If no error don't print output +- docker build . -t matss > /dev/null +script: +# Load packages and install the required packages +# Install retriever from source and run tests +# - docker run matss /bin/sh -c 'Rscript load_and_test.R' - # Exports - - export PYTHON_CONFIG=python${R_PYTHON_VERSION}-config - - export PKG_LIBS=`$PYTHON_CONFIG --ldflags` - - export PKG_CPPFLAGS=`$PYTHON_CONFIG --includes` - - export PKG_CPPFLAGS+=" -D PYTHONLIBFILE=\\\"libpython${R_PYTHON_VERSION}.so\\\"" +- docker run matss Rscript load_and_test.R -scritp: - - retriever ls - - retriever --version -matrix: - include: - - r: devel - - r: release - after_success: - - R CMD INSTALL . - - Rscript -e 'pkgdown::build_site()' - - Rscript -e 'covr::codecov()' - deploy: - provider: pages - skip-cleanup: true - github-token: $GITHUB_PAT - keep-history: true - local-dir: docs - on: - branch: master +# matrix: +# include: +# - r: devel +# - r: release +# after_success: +# - R CMD INSTALL . +# - Rscript -e 'pkgdown::build_site()' +# - Rscript -e 'covr::codecov()' +# deploy: +# provider: pages +# skip-cleanup: true +# github-token: $GITHUB_PAT +# keep-history: true +# local-dir: docs +# on: +# branch: master diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1a90a9f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,63 @@ +FROM rocker/tidyverse:latest + +MAINTAINER Weecology "https://github.com/weecology/rdataretriever" + +# Write enviromental options to config files +RUN echo "options(repos='https://cran.mtu.edu/')" >> ~/.Rprofile +RUN echo "options(repos='https://cran.mtu.edu/')" >> ~/.Renviron +RUN echo "R_LIBS=\"/usr/lib/R/library\"">> ~/.Rprofile +RUN echo "R_LIBS=\"/usr/lib/R/library\"">> ~/.Renviron +RUN echo "R_LIBS_USER=\"/usr/lib/R/library\"">> ~/.Renviron + +RUN apt-get update +RUN apt-get install -y build-essential --allow-downgrades --allow-remove-essential --allow-change-held-packages +RUN apt-get install -y wget git locales locales-all --allow-downgrades --allow-remove-essential --allow-change-held-packages +RUN apt-get install -y postgresql-client mysql-client --allow-downgrades --allow-remove-essential --allow-change-held-packages + +RUN apt-get install -y libgsl0-dbg libgsl0-dev +RUN apt-get install -y libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev + +# RUN apt-get install --allow libgsl0ldbl # not available + +# Set encoding +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + +# Remove python2 and install python3 +RUN apt-get remove -y python && apt-get install -y python3 python3-pip curl +RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python +RUN rm -f /usr/bin/pip && ln -s /usr/bin/pip3 /usr/bin/pip + + + +RUN echo "export PATH="/usr/bin/python:$PATH"" >> ~/.profile +RUN echo "export PYTHONPATH="/usr/bin/python:$PYTHONPATH"" >> ~/.profile +RUN echo "export PGPASSFILE="~/.pgpass"" >> ~/.profile + +# Add permissions to config files +RUN chmod 0644 ~/.Renviron +RUN chmod 0644 ~/.Rprofile +RUN chmod 0644 ~/.profile + +# Install retriever python package +RUN pip install git+https://git@github.com/weecology/retriever.git +RUN retriever ls > /dev/null +RUN pip install psycopg2 pymysql > /dev/null +RUN R_RETICULATE_PYTHON="/usr/bin/python" | echo $R_RETICULATE_PYTHON >> ~/.Renviron + +COPY . ./MATSS +# Use entrypoint to run more configurations. +# set permissions. +# entrypoint.sh will set out config files +# RUN chmod 0755 MATSS/cli_tools/entrypoint.sh +# ENTRYPOINT ["/cli_tools/entrypoint.sh"] + +WORKDIR ./MATSS + +# Change permissions to config files +# Do not run these cmds before Entrypoint. +# RUN chmod 600 cli_tools/.pgpass +# RUN chmod 600 cli_tools/.my.cnf + +CMD ["bash", "-c", "retriever -v"] diff --git a/cli_tools/.my.cnf b/cli_tools/.my.cnf new file mode 100644 index 0000000..0757521 --- /dev/null +++ b/cli_tools/.my.cnf @@ -0,0 +1,5 @@ +[client] +user="travis" +password="Password12!" +host="mysqldb" +port="3306" diff --git a/cli_tools/.pgpass b/cli_tools/.pgpass new file mode 100644 index 0000000..7c1a588 --- /dev/null +++ b/cli_tools/.pgpass @@ -0,0 +1 @@ +pgdb:*:testdb:postgres:Password12! \ No newline at end of file diff --git a/cli_tools/entrypoint.sh b/cli_tools/entrypoint.sh new file mode 100644 index 0000000..fd68118 --- /dev/null +++ b/cli_tools/entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +# Copy config files to $HOME +cp -r /MATSS/cli_tools/.pgpass ~/ +cp -r /MATSS/cli_tools/.my.cnf ~/ + +exec "$@" diff --git a/load_and_test.R b/load_and_test.R new file mode 100644 index 0000000..580f1c9 --- /dev/null +++ b/load_and_test.R @@ -0,0 +1,17 @@ +# Used for continous testing platform +# Install analysis packages using pacman +# Pacman will load the packages and install +# the packaes not available + +# Install pacman if it isn't already installed +if ("pacman" %in% rownames(installed.packages()) == FALSE) install.packages("pacman") +suppressMessages(install.packages("RMariaDB", quiet = TRUE)) +suppressMessages( + pacman::p_load(devtools, RCurl, readr, rmarkdown, + testthat, tidyverse, DBI, RPostgreSQL, + RSQLite, reticulate, devtools, RMariaDB) +) + +install.packages(".", repos = NULL, type="source") +# Test package +test_dir("tests/testthat", reporter = c("check", "progress")) From 58ad35c4b13b0111c3051e4f400dfd95a2458fc0 Mon Sep 17 00:00:00 2001 From: henrykironde Date: Fri, 1 Mar 2019 20:42:58 -0500 Subject: [PATCH 3/4] Change deps --- DESCRIPTION | 17 +++++++------ cli_tools/entrypoint.sh | 12 +++++----- load_and_test.R | 32 +++++++++++++++++++++---- tests/testthat/test-02-datasets.R | 6 ++--- tests/testthat/test-03-retriever-data.R | 2 +- 5 files changed, 48 insertions(+), 21 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index ac30f0d..bc80316 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -37,22 +37,25 @@ Imports: dplyr, tidyr, rdataretriever, + reticulate, RSQLite, forecast, networkD3, here, tibble, - rlang + rlang, + usethis Suggests: testthat, knitr, - rmarkdown, - pkgdown, - covr, - reticulate + rmarkdown, + pkgdown, + covr RoxygenNote: 6.1.1 VignetteBuilder: knitr -Remotes: +Remotes: weecology/LDATS, weecology/portalr, - ropensci/drake + ropensci/drake, + ropensci/rdataretriever, + r-lib/usethis diff --git a/cli_tools/entrypoint.sh b/cli_tools/entrypoint.sh index fd68118..2989c50 100644 --- a/cli_tools/entrypoint.sh +++ b/cli_tools/entrypoint.sh @@ -1,8 +1,8 @@ -#!/bin/sh -set -e +# #!/bin/sh +# set -e -# Copy config files to $HOME -cp -r /MATSS/cli_tools/.pgpass ~/ -cp -r /MATSS/cli_tools/.my.cnf ~/ +# # Copy config files to $HOME +# cp -r /MATSS/cli_tools/.pgpass ~/ +# cp -r /MATSS/cli_tools/.my.cnf ~/ -exec "$@" +# exec "$@" diff --git a/load_and_test.R b/load_and_test.R index 580f1c9..5ada99f 100644 --- a/load_and_test.R +++ b/load_and_test.R @@ -3,15 +3,39 @@ # Pacman will load the packages and install # the packaes not available +library(devtools) # Install pacman if it isn't already installed if ("pacman" %in% rownames(installed.packages()) == FALSE) install.packages("pacman") -suppressMessages(install.packages("RMariaDB", quiet = TRUE)) +# suppressMessages(install.packages("RMariaDB", quiet = TRUE)) suppressMessages( pacman::p_load(devtools, RCurl, readr, rmarkdown, - testthat, tidyverse, DBI, RPostgreSQL, - RSQLite, reticulate, devtools, RMariaDB) + testthat, tidyverse, RSQLite, + # DBI, RPostgreSQL, + magrittr, ggplot2, lubridate, + dplyr, tidyr, RSQLite, forecast, + networkD3, here, tibble, rlang) ) +# Install GitHub packages + +devtools::install_github("weecology/portalr") +devtools::install_github("ropensci/drake") +devtools::install_github("rstudio/reticulate") +devtools::install_github("ropensci/rdataretriever") +devtools::install_github("r-lib/usethis") +devtools::install_github("weecology/LDATS") +library(devtools) +library(portalr) +library(drake) +library(reticulate) +library(rdataretriever) +library(usethis) +library(LDATS) install.packages(".", repos = NULL, type="source") # Test package -test_dir("tests/testthat", reporter = c("check", "progress")) +# test_dir("tests/testthat", reporter = c("check", "progress")) + +# Try old way +# This will not work well for reporting and fail/pass +devtools::test() + diff --git a/tests/testthat/test-02-datasets.R b/tests/testthat/test-02-datasets.R index a4e88c3..b8413c9 100644 --- a/tests/testthat/test-02-datasets.R +++ b/tests/testthat/test-02-datasets.R @@ -15,11 +15,11 @@ test_that("Jornada data is retrievable and works", { test_that("Shortgrass Steppe data is retrievable and works", { expect_error(sgs_data <- get_sgs_data(), NA) expect_true(check_data_format(sgs_data)) - expect_known_hash(sgs_data, "ccbdc633c1") + expect_known_hash(sgs_data, "08c3041cd4") }) test_that("Maizuru data is retrievable and works", { expect_error(maizuru_data <- get_maizuru_data(), NA) expect_true(check_data_format(maizuru_data)) - expect_known_hash(maizuru_data, "c93bd061db") -}) + expect_known_hash(maizuru_data, "ccbdc633c1") +}) \ No newline at end of file diff --git a/tests/testthat/test-03-retriever-data.R b/tests/testthat/test-03-retriever-data.R index 0f88f70..901246f 100644 --- a/tests/testthat/test-03-retriever-data.R +++ b/tests/testthat/test-03-retriever-data.R @@ -25,7 +25,7 @@ test_that("retriever downloading and importing work", { expect_error(install_retriever_data("veg-plots-sdl"), NA) expect_error(dat <- import_retriever_data("veg-plots-sdl"), NA) expect_known_hash(dat$veg_plots_sdl_Count1906, "4f6b34f60a") - expect_known_hash(dat$veg_plots_sdl_Photo_info, "5fa488da2a") + expect_known_hash(dat$veg_plots_sdl_Photo_info, "5f85a913f4") expect_known_hash(dat$veg_plots_sdl_Plot_corners, "53e7e72ee9") expect_known_hash(dat$veg_plots_sdl_Plots, "b0c6ec4b6e") expect_known_hash(dat$veg_plots_sdl_Seedling_counts, "dd0d14c55a") From a4fdddecb20ca9caf28393ce20efb0df4218c330 Mon Sep 17 00:00:00 2001 From: henrykironde Date: Fri, 1 Mar 2019 23:09:44 -0500 Subject: [PATCH 4/4] Use this version >= 1.4.0.9000 --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index bc80316..bcd34a7 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -44,7 +44,7 @@ Imports: here, tibble, rlang, - usethis + usethis (>= 1.4.0.9000) Suggests: testthat, knitr,