workflow:

rules:

- if: '$CI_PIPELINE_SOURCE=="schedule" && $CI_COMMIT_REF_NAME=="master"' ## nightly scheduled pipeline at 4:15 UTC

- if: '$CI_PIPELINE_SOURCE=="web"' ## manually started from web UI

- if: '$CI_PIPELINE_SOURCE=="push" && $CI_COMMIT_REF_NAME!="master"' ## branches pushed to GL directly, mirror is set for master branch only

variables:

CRAN_MIRROR: "https://cloud.r-project.org"

_R_CHECK_FORCE_SUGGESTS_: "false"

_R_CHECK_NO_STOP_ON_TEST_ERROR_: "true"

_R_CHECK_SYSTEM_CLOCK_: "false" ## https://stackoverflow.com/questions/63613301/r-cmd-check-note-unable-to-verify-current-time

_R_CHECK_TESTS_NLINES_: "0"

TZ: "UTC" ## Setting TZ for all GLCI jobs to isolate them from timezone. We could have a new GLCI job to test under

## a non-UTC timezone, although, that's what we do routinely in dev.

RUN_ALL_DATATABLE_TESTS: "yes" ## run optional tests in CI

R_REL_VERSION: "4.5" # only raise when RTOOLS for REL is available

R_REL_WIN_BIN: "https://cloud.r-project.org/bin/windows/base/old/4.5.0/R-4.5.0-win.exe"

R_DEV_VERSION: "4.7"

R_DEV_WIN_BIN: "https://cloud.r-project.org/bin/windows/base/R-devel-win.exe"

R_OLD_VERSION: "4.4"

R_OLD_WIN_BIN: "https://cloud.r-project.org/bin/windows/base/old/4.4.3/R-4.4.3-win.exe"

R_REL_MAC_BIN: "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.5.0-arm64.pkg"

R_OLD_MAC_BIN: "https://cloud.r-project.org/bin/macosx/big-sur-arm64/base/R-4.4.3-arm64.pkg"

stages:

- dependencies

- build

- test

- integration

- deploy

.artifacts-template: &artifacts

artifacts:

expire_in: 2 weeks

when: always

paths:

- bus/$CI_JOB_NAME

mirror-packages:

stage: dependencies

tags:

- saas-linux-medium-amd64

image: registry.gitlab.com/rdatatable/dockerfiles/r-base-minimal

cache:

paths:

- bus/$CI_JOB_NAME/cran

script:

- echo 'source(".ci/ci.R")' >> .Rprofile

- mkdir -p bus/$CI_JOB_NAME/cran/src/contrib

- Rscript -e 'mirror.packages(dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran")'

- Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_DEV_VERSION","R_OLD_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'

- Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_REL_VERSION","R_OLD_VERSION"))), function(binary.ver) mirror.packages(type="mac.binary.big-sur-arm64", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'

<<: *artifacts

.test-install-deps: &install-deps

- Rscript -e 'source(".ci/ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"), repos=file.path("file:", normalizePath("bus/mirror-packages/cran", mustWork=FALSE)), quiet=TRUE)'

build:

stage: build

tags:

- saas-linux-medium-amd64

image: registry.gitlab.com/rdatatable/dockerfiles/r-base-gcc

needs: ["mirror-packages"]

before_script:

- *install-deps

- rm -r bus

script:

- sed -i '/^[[:space:]]*$/d' ./DESCRIPTION ## make last line end abruptly; i.e. without a final \n

- echo "Revision:" $CI_COMMIT_SHA >> ./DESCRIPTION

- R CMD build .

- mkdir -p bus/$CI_JOB_NAME/

- mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME/

<<: *artifacts

.test-template: &test

stage: test

needs: ["mirror-packages","build"]

allow_failure: true

<<: *artifacts

.test-lin-template: &test-lin

<<: *test

tags:

- saas-linux-medium-amd64

before_script:

- cp $(ls -1t bus/build/data.table_*.tar.gz | head -n 1) .

- mkdir -p ~/.R

after_script:

- mkdir -p bus/$CI_JOB_NAME

- echo $CI_JOB_ID > bus/$CI_JOB_NAME/id

- echo $CI_JOB_STATUS > bus/$CI_JOB_NAME/status

- echo $CI_JOB_IMAGE > bus/$CI_JOB_NAME/image

- '[ -d data.table.Rcheck ] && mv data.table.Rcheck bus/$CI_JOB_NAME/'

test-lin-rel:

<<: *test-lin

image: registry.gitlab.com/rdatatable/dockerfiles/r-data.table

variables:

_R_CHECK_FORCE_SUGGESTS_: "TRUE"

OPENBLAS_MAIN_FREE: "1"

_R_CHECK_RD_CHECKRD_MINLEVEL_: "-Inf"

script:

- *install-deps

- echo 'CFLAGS=-g -O3 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars

- echo 'CXXFLAGS=-g -O3 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars

- echo '_R_CHECK_COMPILATION_FLAGS_KNOWN_=-Wvla' >> ~/.Renviron

- |

test-lin-rel-vanilla:

<<: *test-lin

image: registry.gitlab.com/rdatatable/dockerfiles/r-base-gcc

variables:

_R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"

script:

- echo 'CFLAGS=-g -O0 -fno-openmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars

- echo 'CXXFLAGS=-g -O0 -fno-openmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars

- R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)

test-lin-rel-cran:

<<: *test-lin

image: registry.gitlab.com/rdatatable/dockerfiles/r-base

variables:

_R_CHECK_CRAN_INCOMING_: "TRUE" ## stricter --as-cran checks should run in dev pipelines continuously (not sure what they are though)

_R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE" ## Other than no URL checking (takes many minutes) or 'Days since last update 0' NOTEs needed, #3284

_R_CHECK_CRAN_INCOMING_TARBALL_THRESHOLD_: "7500000" ## bytes

_R_CHECK_PKG_SIZES_THRESHOLD_: "10" ## MiB 'checking installed package size' NOTE increased due to po

script:

- *install-deps

- echo 'CFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars

- echo 'CXXFLAGS=-g -O2 -fopenmp -Wall -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars

- |

test-lin-dev-gcc-strict-cran:

<<: *test-lin

image: registry.gitlab.com/rdatatable/dockerfiles/r-devel-gcc-strict

variables:

_R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"

_R_CHECK_CRAN_INCOMING_: "TRUE"

_R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"

_R_S3_METHOD_LOOKUP_BASEENV_AFTER_GLOBALENV_: "FALSE" ## detects S3 method lookup found on search path #4777

_R_S3_METHOD_LOOKUP_REPORT_SEARCH_PATH_USES_: "TRUE"

script:

- echo 'CFLAGS=-g -O2 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars

- echo 'CXXFLAGS=-g -O2 -flto=auto -fno-common -fopenmp -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars

- *install-deps

- R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)

- (! grep "warning:" data.table.Rcheck/00install.out)

- >-

test-lin-dev-clang-cran:

<<: *test-lin

image: registry.gitlab.com/rdatatable/dockerfiles/r-devel-clang

variables:

_R_CHECK_COMPILATION_FLAGS_KNOWN_: "-Wvla"

_R_CHECK_CRAN_INCOMING_: "TRUE"

_R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"

_R_S3_METHOD_LOOKUP_BASEENV_AFTER_GLOBALENV_: "FALSE"

_R_S3_METHOD_LOOKUP_REPORT_SEARCH_PATH_USES_: "TRUE"

script:

- echo 'CFLAGS=-g -O2 -fno-common -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' > ~/.R/Makevars

- echo 'CXXFLAGS=-g -O2 -fno-common -Wall -Wvla -pedantic -fstack-protector-strong -D_FORTIFY_SOURCE=2' >> ~/.R/Makevars

- *install-deps

- clang-tidy -extra-arg=-I/usr/local/lib/R/include -checks='readability-inconsistent-declaration-parameter' src/*.c -- -std=c99

- R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)

- (! grep "warning:" data.table.Rcheck/00install.out)

- >-

test-lin-ancient-cran:

<<: *test-lin

image: registry.gitlab.com/rdatatable/dockerfiles/r-3.5.0

variables:

_R_CHECK_FORCE_SUGGESTS_: "FALSE" # can be removed if all dependencies are available (knitr, xts, etc.)

script:

- *install-deps

# knitr requires evaluate, which requires R 3.6.0.

# Restore checking vignettes if upgrading our R dependency means knitr can be installed, or when we switch to litedown.

- R CMD check --no-manual --no-build-vignettes --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)

test-lin-dev-clang-san:

<<: *test-lin

image: registry.gitlab.com/rdatatable/dockerfiles/r-devel-clang-san

variables:

# must be set for most of the process because there are pseudo-leaks everywhere

ASAN_OPTIONS: "detect_leaks=0"

# fontconfig is known to leak; add more suppressions as discovered

LSAN_OPTIONS: "suppressions=$CI_PROJECT_DIR/.dev/lsan.supp"

UBSAN_OPTIONS: "print_stacktrace=1"

script:

- *install-deps

- |

test-lin-dev-gcc-san:

<<: *test-lin

image: registry.gitlab.com/rdatatable/dockerfiles/r-devel-gcc-san

variables:

# must be set for most of the process because there are pseudo-leaks everywhere

ASAN_OPTIONS: "detect_leaks=0"

# fontconfig is known to leak; add more suppressions as discovered

LSAN_OPTIONS: "suppressions=$CI_PROJECT_DIR/.dev/lsan.supp"

UBSAN_OPTIONS: "print_stacktrace=1"

script:

- *install-deps

- |

.test-win-template: &test-win

<<: *test

tags:

- saas-windows-medium-amd64

before_script:

- Invoke-WebRequest -Uri $R_BIN -OutFile ../R-win.exe -ErrorAction Stop; if (!(Test-Path -Path ..\R-win.exe)) {Write-Error "R-win.exe not found, download failed?"}

- Start-Process -FilePath ..\R-win.exe -ArgumentList "/VERYSILENT /DIR=C:\R" -NoNewWindow -Wait

- $VERSION = $RTOOLS_VERSION -replace '\.', ''

# persist RTOOLSXX_HOME="/c/rtools" which is needed for finding gcc #6552

- Add-Content -Path ".\.Renviron" -Value "RTOOLS${VERSION}_HOME='/c/rtools'"

- $RTOOLS_BIN="https://github.com/r-hub/rtools${VERSION}/releases/download/latest/rtools${VERSION}.exe"

- Invoke-WebRequest -Uri $RTOOLS_BIN -OutFile ../rtools.exe -ErrorAction Stop; if (!(Test-Path -Path ..\rtools.exe)) {Write-Error "rtools.exe not found, download failed?"}

- Start-Process -FilePath ..\rtools.exe -ArgumentList "/VERYSILENT /DIR=C:\rtools" -NoNewWindow -Wait

- $env:PATH = "C:\R\bin;C:\rtools\usr\bin;$env:PATH"

- Rscript.exe -e "source('.ci/ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'), repos=file.path('file://',getwd(),'bus/mirror-packages/cran'), quiet=TRUE)"

- cp.exe $(ls.exe -1t bus/build/data.table_*.tar.gz | head.exe -n 1) .

script:

- R.exe CMD check --no-manual $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)

- R.exe CMD INSTALL --build $(ls.exe -1t data.table_*.tar.gz | head.exe -n 1)

- -not (grep.exe "warning:" data.table.Rcheck\00install.out)

after_script:

- $env:PATH = "C:\R\bin;C:\rtools\usr\bin;$env:PATH"

- mkdir.exe -p bus/$CI_JOB_NAME

- Rscript.exe -e "cat(Sys.getenv('CI_JOB_ID'), file=file.path('bus', Sys.getenv('CI_JOB_NAME'), 'id'))"

- Rscript.exe -e "cat(Sys.getenv('CI_JOB_STATUS'), file=file.path('bus', Sys.getenv('CI_JOB_NAME'), 'status'))"

- Rscript.exe -e "cat(Sys.getenv('CI_JOB_IMAGE'), file=file.path('bus', Sys.getenv('CI_JOB_NAME'), 'image'))"

- Rscript.exe -e "to<-file.path('bus', Sys.getenv('CI_JOB_NAME'), 'data.table.Rcheck'); if (dir.exists(from<-'data.table.Rcheck')) invisible(file.rename(from, to)); dir.exists(to)"

- Rscript.exe -e "from<-tail(list.files(pattern='^data\\.table_.*\\.zip$'), 1L); to<-file.path('bus', Sys.getenv('CI_JOB_NAME'), from); if (length(from)) invisible(file.rename(from, to)); length(to)&&file.exists(to)"

test-win-rel:

<<: *test-win

variables:

R_BIN: "$R_REL_WIN_BIN"

RTOOLS_VERSION: "$R_REL_VERSION"

test-win-dev:

<<: *test-win

variables:

R_BIN: "$R_DEV_WIN_BIN"

RTOOLS_VERSION: "$R_REL_VERSION"

test-win-old:

<<: *test-win

variables:

R_BIN: "$R_OLD_WIN_BIN"

RTOOLS_VERSION: "$R_OLD_VERSION"

.test-mac-template: &test-mac

<<: *test

tags:

- saas-macos-medium-m1

before_script:

- curl -LO https://github.com/XQuartz/XQuartz/releases/download/XQuartz-2.8.5/XQuartz-2.8.5.pkg

- sudo installer -pkg "$(ls -1t XQuartz-*.pkg | head -n 1)" -target /

- curl -O $R_BIN

- sudo installer -pkg "$(ls -1t R-*-arm64.pkg | head -n 1)" -target /

- sudo Rscript -e "source('https://mac.R-project.org/bin/install.R'); install.libs('gettext')"

- *install-deps

- cp $(ls -1t bus/build/data.table_*.tar.gz | head -n 1) .

script:

- R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)

- R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)

- >-

after_script:

- mkdir -p bus/$CI_JOB_NAME

- '[ -d data.table.Rcheck ] && mv data.table.Rcheck bus/$CI_JOB_NAME/'

- '[ -f data.table_*.tgz ] && mv $(ls -1t data.table_*.tgz | head -n 1) bus/$CI_JOB_NAME/'

- echo $CI_JOB_ID > bus/$CI_JOB_NAME/id

- echo $CI_JOB_STATUS > bus/$CI_JOB_NAME/status

- echo $CI_JOB_IMAGE > bus/$CI_JOB_NAME/image

test-mac-rel:

<<: *test-mac

image: macos-15-xcode-16

variables:

R_VERSION: "$R_REL_VERSION"

R_BIN: "$R_REL_MAC_BIN"

test-mac-old:

<<: *test-mac

image: macos-15-xcode-16

variables:

R_VERSION: "$R_OLD_VERSION"

R_BIN: "$R_OLD_MAC_BIN"

integration:

stage: integration

image: registry.gitlab.com/rdatatable/dockerfiles/r-data.table

tags:

- saas-linux-medium-amd64

rules:

- if: '$CI_COMMIT_REF_NAME == "master"'

- if: '$CI_PIPELINE_SOURCE == "web"'

needs: ["mirror-packages","build","test-lin-rel","test-lin-rel-cran","test-lin-dev-gcc-strict-cran","test-lin-dev-clang-cran","test-lin-rel-vanilla","test-lin-ancient-cran","test-lin-dev-clang-san","test-lin-dev-gcc-san","test-win-rel","test-win-dev" ,"test-win-old","test-mac-rel","test-mac-old"]

script:

- R --version

- *install-deps ## markdown pkg not present in r-litedown image

## Install data.table from build artifacts (needed by litedown site generation)

- R CMD INSTALL bus/build/data.table_*.tar.gz

## Copy assets to site directory

- mkdir -p ./site/assets/favicon/

- cp .graphics/favicon/* ./site/assets/favicon/

## Generate reference pages first (creates reference_index.rds needed by manual.Rmd)

- Rscript site/generate_reference.R

## Build litedown site (now manual.Rmd can use reference_index.rds)

- cd site && Rscript -e 'litedown::fuse_site()' && cd ..

- mkdir -p website

- cp -r site/*.html website/

- cp -r site/reference website/ 2>/dev/null || true

- cp -r site/assets website/

## html manual, vignettes, repos, cran_web, cran_checks

- echo 'source(".ci/ci.R"); source(".ci/publish.R")' >> .Rprofile

## list of available test-* jobs dynamically based on bus/test-* directories

- Rscript -e 'cat("\ntest.jobs <- c(\n"); cat(paste0(" \"",list.files("bus",pattern="^test-"),"\" = \"data.table\""), sep=",\n"); cat(")\n")' >> .Rprofile

- Rscript -e 'sapply(names(test.jobs), check.test, pkg="data.table", simplify=FALSE)'

- mkdir -p bus/$CI_JOB_NAME

## delete any existing non-dev version of data.table

- rm -f bus/mirror-packages/cran/src/contrib/data.table_*.tar.gz

- rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_REL_VERSION/data.table_*.zip

- rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_DEV_VERSION/data.table_*.zip

- rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_OLD_VERSION/data.table_*.zip

- rm -f bus/mirror-packages/cran/bin/macosx/big-sur-arm64/contrib/$R_REL_VERSION/data.table_*.tgz

# - rm -f bus/mirror-packages/cran/bin/macosx/big-sur-arm64/contrib/$R_DEV_VERSION/data.table_*.tgz

- rm -f bus/mirror-packages/cran/bin/macosx/big-sur-arm64/contrib/$R_OLD_VERSION/data.table_*.tgz

## merge mirror-packages and R devel packages

- mv bus/mirror-packages/cran bus/$CI_JOB_NAME/

## publish package sources

- mkdir -p bus/$CI_JOB_NAME/cran/library bus/$CI_JOB_NAME/cran/doc

- mv $(ls -1t bus/build/data.table_*.tar.gz | head -n 1) bus/$CI_JOB_NAME/cran/src/contrib

- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="source"), type="source", fields="Revision", addFiles=TRUE)'

## publish binaries

- mkdir -p bus/integration/cran/bin/windows/contrib/$R_REL_VERSION/

- mkdir -p bus/integration/cran/bin/windows/contrib/$R_DEV_VERSION/

- mkdir -p bus/integration/cran/bin/windows/contrib/$R_OLD_VERSION/

- '[ -f bus/test-win-rel/data.table_*.zip ] && cp bus/test-win-rel/data.table_*.zip bus/integration/cran/bin/windows/contrib/$R_REL_VERSION/'

- ls -1 "bus/integration/cran/bin/windows/contrib/$R_REL_VERSION"/data.table_*.zip || true

- '[ -f bus/test-win-dev/data.table_*.zip ] && cp bus/test-win-dev/data.table_*.zip bus/integration/cran/bin/windows/contrib/$R_DEV_VERSION/'

- ls -1 "bus/integration/cran/bin/windows/contrib/$R_DEV_VERSION"/data.table_*.zip || true

- '[ -f bus/test-win-old/data.table_*.zip ] && cp bus/test-win-old/data.table_*.zip bus/integration/cran/bin/windows/contrib/$R_OLD_VERSION/'

- ls -1 "bus/integration/cran/bin/windows/contrib/$R_OLD_VERSION"/data.table_*.zip || true

- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_REL_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'

- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_DEV_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'

- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_OLD_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'

#### macos mkdir cran/bin/.../contrib/...

- mkdir -p bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_REL_VERSION/

# - mkdir -p bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_DEV_VERSION/

- mkdir -p bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_OLD_VERSION/

#### macos move binaries

- '[ -f bus/test-mac-rel/data.table_*.tgz ] && cp bus/test-mac-rel/data.table_*.tgz bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_REL_VERSION/'

- ls -1 "bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_REL_VERSION"/data.table_*.tgz || true

# - '[ -f bus/test-mac-dev/data.table_*.tgz ] && cp bus/test-mac-dev/data.table_*.tgz bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_DEV_VERSION/'

# - ls -1 "bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_DEV_VERSION"/data.table_*.tgz || true

- '[ -f bus/test-mac-old/data.table_*.tgz ] && cp bus/test-mac-old/data.table_*.tgz bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_OLD_VERSION/'

- ls -1 "bus/integration/cran/bin/macosx/big-sur-arm64/contrib/$R_OLD_VERSION"/data.table_*.tgz || true

#### macos write_PACKAGES

- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.big-sur-arm64", ver=Sys.getenv("R_REL_VERSION")), type="mac.binary", fields="Revision", addFiles=TRUE)'

# - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.big-sur-arm64", ver=Sys.getenv("R_DEV_VERSION")), type="mac.binary", fields="Revision", addFiles=TRUE)'

- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.big-sur-arm64", ver=Sys.getenv("R_OLD_VERSION")), type="mac.binary", fields="Revision", addFiles=TRUE)'

## install pkg to render html

- mkdir -p /tmp/opencran/library /tmp/opencran/doc/html

- Rscript -e 'install.packages("data.table", lib="/tmp/opencran/library", repos=file.path("file:",normalizePath("bus/integration/cran")), INSTALL_opts="--html", quiet=TRUE)'

- Rscript -e 'packageVersion("data.table", lib.loc="/tmp/opencran/library")'

## CRAN style web/CRAN_web.css

- wget -q -P bus/integration/cran/web https://cran.r-project.org/web/CRAN_web.css

## web/packages/$pkg/index.html

- Rscript -e 'sapply(setNames(nm=rownames(installed.packages(lib.loc="/tmp/opencran/library", priority="NA"))), package.index, lib.loc="/tmp/opencran/library")'

## R docs, html, css, icons

- Rscript -e 'doc.copy(repodir="/tmp/opencran")'

## Update packages.html, fix paths

- Rscript -e 'setwd("/tmp/opencran/doc/html"); make.packages.html(lib.loc="../../library", docdir="/tmp/opencran/doc"); tmp<-readLines(f<-"/tmp/opencran/doc/html/packages.html"); writeLines(gsub("file:///../../library","../../library", tmp, fixed=TRUE), f)'

- mv /tmp/opencran/doc bus/integration/cran/

## library html manual, vignettes

- Rscript -e 'lib.copy(lib.from="/tmp/opencran/library")'

## web/checks/$pkg/$job 00install.out, 00check.log, *.Rout

- Rscript -e 'sapply(names(test.jobs), check.copy)'

## web/packages/$pkg/$pkg.pdf

- Rscript -e 'pdf.copy("data.table", "test-lin-rel")'

## web/checks/check_results_$pkg.html

- Rscript -e 'check.index("data.table", names(test.jobs))'

## web/checks/check_flavors.html

- Rscript -e 'check.flavors(names(test.jobs))'

## Add backwards compatibility redirects

- mkdir -p website/news website/reference website/articles

- echo '<script>location.replace("../news.html")</script>' > website/news/index.html

- echo '<script>location.replace("../manual.html")</script>' > website/reference/index.html

## Render translated vignettes into articles/<language>

- Rscript site/render_translations.R

## Copy existing vignettes into articles/

- cp -r bus/integration/cran/library/data.table/doc/* website/articles/

## litedown merge

- Rscript -e 'common_files<-function(path1, path2) intersect(list.files(path1, all.files=TRUE, no..=TRUE), list.files(path2, all.files=TRUE, no..=TRUE)); msg = if (length(f<-common_files("website","bus/integration/cran"))) paste(c("Following artifacts will be overwritten by litedown artifacts:", paste0(" ", f)), collapse="\n") else "No overlapping files from litedown artifacts"; message(msg); q("no")'

- mv website/* bus/integration/cran/

## add plausible.io stats

- find bus/integration/cran -type f -iname "*.html" | xargs sed -i 's!<head>!<head><script defer data-domain="rdatatable.gitlab.io/data.table" src="https://plausible.io/js/script.js"></script>!g'

<<: *artifacts

pages:

stage: deploy

environment: production

tags:

- saas-linux-medium-amd64

only:

- master

image: ubuntu

needs: ["integration"]

script:

- mkdir -p public

- cp -r bus/integration/cran/* public

- cat public/src/contrib/PACKAGES

artifacts:

expire_in: 2 weeks

paths:

- public