diff --git a/NEWS.md b/NEWS.md index c111497f7..e10b70676 100644 --- a/NEWS.md +++ b/NEWS.md @@ -44,6 +44,8 @@ 11. `setorderv()` now accepts a named vector for the `order` argument. When provided, the names are used to identify the columns, allowing the `cols` argument to be omitted, [#6932](https://github.com/Rdatatable/data.table/issues/6932). Thanks to @MichaelChirico for the suggestion and @venom1204 for the implementation. +12. `nafill()` and `setnafill()` gain a `limit` argument to restrict the maximum number of consecutive `NA` values filled during `locf` or `nocb` operations, [#7677](https://github.com/Rdatatable/data.table/issues/7677). Thanks to @jaynewton for the suggestion and @venom1204 for the PR. + ### BUG FIXES 1. `fread()` with `skip=0` and `(header=TRUE|FALSE)` no longer skips the first row when it has fewer fields than subsequent rows, [#7463](https://github.com/Rdatatable/data.table/issues/7463). Thanks @emayerhofer for the report and @ben-schwen for the fix. diff --git a/R/shift.R b/R/shift.R index 1c68d13c4..024081db6 100644 --- a/R/shift.R +++ b/R/shift.R @@ -26,12 +26,16 @@ shift = function(x, n=1L, fill, type=c("lag", "lead", "shift", "cyclic"), give.n ans } -nafill = function(x, type=c("const","locf","nocb"), fill=NA, nan=NA) { +nafill = function(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, limit=Inf) { type = match.arg(type) - .Call(CnafillR, x, type, fill, nan_is_na(nan), FALSE, NULL) + if (!is.numeric(limit) || length(limit) != 1L || is.na(limit) || limit < 0) + stopf("limit must be a non-negative scalar numeric") + .Call(CnafillR, x, type, fill, nan_is_na(nan), FALSE, NULL, as.double(floor(limit))) } -setnafill = function(x, type=c("const","locf","nocb"), fill=NA, nan=NA, cols=seq_along(x)) { +setnafill = function(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x), limit=Inf) { type = match.arg(type) - invisible(.Call(CnafillR, x, type, fill, nan_is_na(nan), TRUE, cols)) + if (!is.numeric(limit) || length(limit) != 1L || is.na(limit) || limit < 0) + stopf("limit must be a non-negative scalar numeric") + invisible(.Call(CnafillR, x, type, fill, nan_is_na(nan), TRUE, cols, as.double(floor(limit)))) } diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index 6e5382883..f217235f2 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -21952,3 +21952,27 @@ test(2385.04, setorderv(DT, order=c(x=1L, x=-1L)), error="order argument has nam test(2385.05, setorderv(DT, order=c(x=2L)), error="Must be +1 or -1") DT = data.table(a=c(2,1,2), b=3:1) test(2385.06, setorderv(copy(DT), order=c(a=1L)), setorderv(copy(DT), cols="a", order=1L)) + +# #7677: add limit argument to nafill() +test(2380.01, nafill(c(1, NA, NA, NA, 5), type="locf", limit=1), c(1, 1, NA, NA, 5)) +test(2380.02, nafill(c(1, NA, NA, 5, NA, NA, NA, 9), type="locf", limit=2), c(1, 1, 1, 5, 5, 5, NA, 9)) +test(2380.03, nafill(c(1, NA, NA, 4, NA), type="locf", limit=Inf), c(1, 1, 1, 4, 4)) +test(2380.04, nafill(c(NA, NA, 3, NA, NA), type="locf", limit=1), c(NA, NA, 3, 3, NA)) +test(2380.05, nafill(c(1, NA, NA, 5), type="nocb", limit=1), c(1, NA, 5, 5)) +test(2380.06, nafill(c(1L, NA, NA), type="locf", limit=1), c(1L, 1L, NA)) +test(2380.07, nafill(c("a", NA, NA), type="locf", limit=1), c("a", "a", NA)) +test(2380.08, nafill(c(1, NA, NA), type="locf", limit=0), c(1, NA, NA)) +dt = data.table(a=c(1, NA, NA)) +setnafill(dt, type="locf", limit=1) +test(2380.09, {dt=data.table(a=c(1,NA,NA)); setnafill(dt, type="locf", limit=1); dt$a}, c(1, 1, NA)) +test(2380.10, nafill(c(NA, 1), "locf", fill=0, limit=0), c(0, 1)) +test(2380.11, nafill(c(1, NA, NA), type="locf", limit=1.5), c(1, 1, NA)) +test(2380.12, nafill(c(1, NA, NA), type="const", fill=0, limit=1), c(1, 0, 0)) +test(2380.13, nafill(1:5, limit=-1), error="limit must be a non-negative scalar numeric") +test(2380.14, nafill(1:5, limit=NA), error="limit must be a non-negative scalar numeric") +test(2380.15, nafill(1:5, limit=NULL), error="limit must be a non-negative scalar numeric") +test(2380.16, nafill(1:5, limit=c(1, 2)), error="limit must be a non-negative scalar numeric") +test(2380.17, nafill(1:5, limit=1+2i), error="limit must be a non-negative scalar numeric") +test(2380.18, setnafill(data.table(a=1:5), limit=-1), error="limit must be a non-negative scalar numeric") +test(2380.19, nafill(c(1.1, NA, NA, 4.4), type="nocb", limit=1), c(1.1, NA, 4.4, 4.4)) +test(2380.20, setnafill(data.table(a=1:5), limit=-1), error="limit must be a non-negative scalar numeric") diff --git a/man/nafill.Rd b/man/nafill.Rd index 90c4b1c5c..a863d82ed 100644 --- a/man/nafill.Rd +++ b/man/nafill.Rd @@ -10,8 +10,8 @@ Fast fill missing values using constant value, \emph{last observation carried forward} or \emph{next observation carried backward}. } \usage{ -nafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA) -setnafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x)) +nafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, limit=Inf) +setnafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x), limit=Inf) } \arguments{ \item{x}{ Vector, list, data.frame or data.table of logical, numeric or character columns. } @@ -19,6 +19,7 @@ setnafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x) \item{fill}{ Value to be used to replace missing observations. See examples. } \item{nan}{ Either \code{NaN} or \code{NA}; if the former, \code{NaN} is treated as distinct from \code{NA}, otherwise, they are treated the same during replacement. See Examples. } \item{cols}{ Numeric or character vector specifying columns to be updated. } + \item{limit}{The maximum number of consecutive \code{NA} values to fill. Must be a non-negative scalar numeric. Default is \code{Inf}. Fractional values are truncated via \code{floor}. This argument is ignored when \code{type="const"}.} } \details{ Supported types are \emph{logical}, \emph{integer}, \emph{double}, \emph{character}, and \emph{factor}, as well as classes built on top of these such as \code{Date}, \code{IDate}, and \code{POSIXct}. @@ -51,6 +52,13 @@ nafill(dt, "nocb") setnafill(dt, "locf", cols=c("v2","v3")) dt + +# limit= restricts the number of consecutive fills +y = c(1, NA, NA, NA, 5) +nafill(c(NA, 1), "locf", fill=0, limit=0) # Result: 0, 1 (Boundary case) +nafill(y, "locf", limit=1) # Only fills the first NA +nafill(y, "locf", limit=Inf) # Fills all NAs (default) + } \seealso{ \code{\link{shift}}, \code{\link{data.table}}, \code{\link{fcoalesce}} diff --git a/src/data.table.h b/src/data.table.h index df46c4a33..4c4d112df 100644 --- a/src/data.table.h +++ b/src/data.table.h @@ -311,9 +311,11 @@ SEXP copyAsGrowable(SEXP x); SEXP resizeVector(SEXP x, SEXP size); // nafill.c -void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose); -void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose); -SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols); +void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose, uint_fast64_t limit); +void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose, uint_fast64_t limit); +void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fill, ans_t *ans, bool verbose, uint_fast64_t limit); +void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, ans_t *ans, bool verbose, uint_fast64_t limit); +SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols, SEXP limit); // between.c SEXP between(SEXP x, SEXP lower, SEXP upper, SEXP incbounds, SEXP NAbounds, SEXP check); diff --git a/src/nafill.c b/src/nafill.c index 8a8e8eab5..ac46735eb 100644 --- a/src/nafill.c +++ b/src/nafill.c @@ -1,48 +1,43 @@ #include "data.table.h" -void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose) { +void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose, uint_fast64_t limit) { double tic=0.0; if (verbose) tic = omp_get_wtime(); if (type==0) { // const - if (nan_is_na) { - for (uint_fast64_t i=0; idbl_v[i] = ISNAN(x[i]) ? fill : x[i]; - } - } else { - for (uint_fast64_t i=0; idbl_v[i] = ISNA(x[i]) ? fill : x[i]; - } + for (uint_fast64_t i=0; idbl_v[i] = is_na ? fill : x[i]; } } else if (type==1) { // locf - if (nan_is_na) { - ans->dbl_v[0] = ISNAN(x[0]) ? fill : x[0]; - for (uint_fast64_t i=1; idbl_v[i] = ISNAN(x[i]) ? ans->dbl_v[i-1] : x[i]; - } - } else { - ans->dbl_v[0] = ISNA(x[0]) ? fill : x[0]; - for (uint_fast64_t i=1; idbl_v[i] = ISNA(x[i]) ? ans->dbl_v[i-1] : x[i]; - } + uint_fast64_t fills = 0; + bool is_na = nan_is_na ? ISNAN(x[0]) : ISNA(x[0]); + ans->dbl_v[0] = is_na ? fill : x[0]; + if (is_na) fills = 1; + for (uint_fast64_t i=1; idbl_v[i] = ans->dbl_v[i-1]; fills++; } + else ans->dbl_v[i] = x[i]; + } else { ans->dbl_v[i] = x[i]; fills = 0; } } } else if (type==2) { // nocb - if (nan_is_na) { - ans->dbl_v[nx-1] = ISNAN(x[nx-1]) ? fill : x[nx-1]; - for (int_fast64_t i=nx-2; i>=0; i--) { - ans->dbl_v[i] = ISNAN(x[i]) ? ans->dbl_v[i+1] : x[i]; - } - } else { - ans->dbl_v[nx-1] = ISNA(x[nx-1]) ? fill : x[nx-1]; - for (int_fast64_t i=nx-2; i>=0; i--) { - ans->dbl_v[i] = ISNA(x[i]) ? ans->dbl_v[i+1] : x[i]; - } + uint_fast64_t fills = 0; + bool is_na = nan_is_na ? ISNAN(x[nx-1]) : ISNA(x[nx-1]); + ans->dbl_v[nx-1] = is_na ? fill : x[nx-1]; + if (is_na) fills = 1; + for (int_fast64_t i=nx-2; i>=0; i--) { + is_na = nan_is_na ? ISNAN(x[i]) : ISNA(x[i]); + if (is_na) { + if (fills < limit) { ans->dbl_v[i] = ans->dbl_v[i+1]; fills++; } + else ans->dbl_v[i] = x[i]; + } else { ans->dbl_v[i] = x[i]; fills = 0; } } } if (verbose) snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic); } -void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose) { +void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose, uint_fast64_t limit) { double tic=0.0; if (verbose) tic = omp_get_wtime(); @@ -51,20 +46,30 @@ void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill ans->int_v[i] = x[i]==NA_INTEGER ? fill : x[i]; } } else if (type==1) { // locf + uint_fast64_t fills = 0; ans->int_v[0] = x[0]==NA_INTEGER ? fill : x[0]; + if (x[0]==NA_INTEGER) fills = 1; for (uint_fast64_t i=1; iint_v[i] = x[i]==NA_INTEGER ? ans->int_v[i-1] : x[i]; + if (x[i]==NA_INTEGER) { + if (fills < limit) { ans->int_v[i] = ans->int_v[i-1]; fills++; } + else ans->int_v[i] = x[i]; + } else { ans->int_v[i] = x[i]; fills = 0; } } } else if (type==2) { // nocb + uint_fast64_t fills = 0; ans->int_v[nx-1] = x[nx-1]==NA_INTEGER ? fill : x[nx-1]; + if (x[nx-1]==NA_INTEGER) fills = 1; for (int_fast64_t i=nx-2; i>=0; i--) { - ans->int_v[i] = x[i]==NA_INTEGER ? ans->int_v[i+1] : x[i]; + if (x[i]==NA_INTEGER) { + if (fills < limit) { ans->int_v[i] = ans->int_v[i+1]; fills++; } + else ans->int_v[i] = x[i]; + } else { ans->int_v[i] = x[i]; fills = 0; } } } if (verbose) snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic); } -void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fill, ans_t *ans, bool verbose) { +void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fill, ans_t *ans, bool verbose, uint_fast64_t limit) { double tic=0.0; if (verbose) tic = omp_get_wtime(); @@ -73,21 +78,31 @@ void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fi ans->int64_v[i] = x[i]==NA_INTEGER64 ? fill : x[i]; } } else if (type==1) { // locf + uint_fast64_t fills = 0; ans->int64_v[0] = x[0]==NA_INTEGER64 ? fill : x[0]; + if (x[0]==NA_INTEGER64) fills = 1; for (uint_fast64_t i=1; iint64_v[i] = x[i]==NA_INTEGER64 ? ans->int64_v[i-1] : x[i]; + if (x[i]==NA_INTEGER64) { + if (fills < limit) { ans->int64_v[i] = ans->int64_v[i-1]; fills++; } + else ans->int64_v[i] = x[i]; + } else { ans->int64_v[i] = x[i]; fills = 0; } } } else if (type==2) { // nocb + uint_fast64_t fills = 0; ans->int64_v[nx-1] = x[nx-1]==NA_INTEGER64 ? fill : x[nx-1]; + if (x[nx-1]==NA_INTEGER64) fills = 1; for (int_fast64_t i=nx-2; i>=0; i--) { - ans->int64_v[i] = x[i]==NA_INTEGER64 ? ans->int64_v[i+1] : x[i]; + if (x[i]==NA_INTEGER64) { + if (fills < limit) { ans->int64_v[i] = ans->int64_v[i+1]; fills++; } + else ans->int64_v[i] = x[i]; + } else { ans->int64_v[i] = x[i]; fills = 0; } } } if (verbose) snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic); } -void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, ans_t *ans, bool verbose) { +void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, ans_t *ans, bool verbose, uint_fast64_t limit) { double tic=0.0; if (verbose) tic = omp_get_wtime(); @@ -96,16 +111,26 @@ void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, SET_STRING_ELT(ans->char_v, i, x[i]==NA_STRING ? fill : x[i]); } } else if (type==1) { // locf + uint_fast64_t fills = 0; SET_STRING_ELT(ans->char_v, 0, x[0]==NA_STRING ? fill : x[0]); + if (x[0]==NA_STRING) fills = 1; const SEXP* thisans = SEXPPTR_RO(ans->char_v); // takes out STRING_ELT from loop for (uint_fast64_t i=1; ichar_v, i, x[i]==NA_STRING ? thisans[i-1] : x[i]); + if (x[i]==NA_STRING) { + if (fills < limit) { SET_STRING_ELT(ans->char_v, i, thisans[i-1]); fills++; } + else SET_STRING_ELT(ans->char_v, i, x[i]); + } else { SET_STRING_ELT(ans->char_v, i, x[i]); fills = 0; } } } else if (type==2) { // nocb + uint_fast64_t fills = 0; SET_STRING_ELT(ans->char_v, nx-1, x[nx-1]==NA_STRING ? fill : x[nx-1]); + if (x[nx-1]==NA_STRING) fills = 1; const SEXP* thisans = SEXPPTR_RO(ans->char_v); // takes out STRING_ELT from loop for (int_fast64_t i=nx-2; i>=0; i--) { - SET_STRING_ELT(ans->char_v, i, x[i]==NA_STRING ? thisans[i+1] : x[i]); + if (x[i]==NA_STRING) { + if (fills < limit) { SET_STRING_ELT(ans->char_v, i, thisans[i+1]); fills++; } + else SET_STRING_ELT(ans->char_v, i, x[i]); + } else { SET_STRING_ELT(ans->char_v, i, x[i]); fills = 0; } } } if (verbose) @@ -117,7 +142,7 @@ void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, over columns of the input data. This includes handling different data types and applying the designated filling method to each column in parallel. */ -SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols) { +SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols, SEXP limit) { int protecti=0; const bool verbose = GetVerbose(); @@ -128,6 +153,9 @@ SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, S if (verbose) tic = omp_get_wtime(); + const double limit_d = REAL(limit)[0]; + const uint_fast64_t limit_n = !R_FINITE(limit_d) || limit_d >= (double)UINT_FAST64_MAX ? UINT_FAST64_MAX : (uint_fast64_t)limit_d; + bool copy = !LOGICAL(inplace)[0]; if (!IS_TRUE_OR_FALSE(nan_is_na_arg)) error(_("'%s' must be TRUE or FALSE"), "nan_is_na"); // # nocov @@ -186,21 +214,7 @@ SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, S for (R_len_t i=0; i