Skip to content

Add limit parameter to nafill and setnafill #7677 #7819

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
venom1204 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add limit parameter to nafill and setnafill #7677 #7819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions R/shift.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,16 @@ shift = function(x, n=1L, fill, type=c("lag", "lead", "shift", "cyclic"), give.n
ans
}

nafill = function(x, type=c("const","locf","nocb"), fill=NA, nan=NA) {
nafill = function(x, type=c("const","locf","nocb"), fill=NA, nan=NA, limit=Inf) {
type = match.arg(type)
.Call(CnafillR, x, type, fill, nan_is_na(nan), FALSE, NULL)
if (!is.numeric(limit) || length(limit) != 1L || limit < 0)
stopf("limit must be a non-negative scalar numeric or Inf")
.Call(CnafillR, x, type, fill, nan_is_na(nan), FALSE, NULL, as.double(limit))
}

setnafill = function(x, type=c("const","locf","nocb"), fill=NA, nan=NA, cols=seq_along(x)) {
setnafill = function(x, type=c("const","locf","nocb"), fill=NA, nan=NA, cols=seq_along(x), limit=Inf) {
type = match.arg(type)
invisible(.Call(CnafillR, x, type, fill, nan_is_na(nan), TRUE, cols))
if (!is.numeric(limit) || length(limit) != 1L || limit < 0)
stopf("limit must be a non-negative scalar numeric or Inf")
invisible(.Call(CnafillR, x, type, fill, nan_is_na(nan), TRUE, cols, as.double(limit)))
}
6 changes: 6 additions & 0 deletions inst/tests/tests.Rraw
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21831,3 +21831,9 @@ test(2378.92, .Call(CresizeVector, x, 2:3), error = "must be length 1 non-NA non
test(2378.93, .Call(CresizeVector, x, -2L), error = "must be length 1 non-NA non-negative integer value")
test(2378.94, .Call(CresizeVector, x, NA_integer_), error = "must be length 1 non-NA non-negative integer value")
test(2378.95, .Call(CresizeVector, NULL, 2L), error = "must be a vector")

# #7677: add limit argument to nafill()
test(2379.01, nafill(c(1, NA, NA, NA, 5), type="locf", limit=1), c(1, 1, NA, NA, 5))
test(2379.02, nafill(c(1, NA, NA, 5, NA, NA, NA, 9), type="locf", limit=2), c(1, 1, 1, 5, 5, 5, NA, 9))
test(2379.03, nafill(c(1, NA, NA, 4, NA), type="locf", limit=Inf), c(1, 1, 1, 4, 4))
test(2379.04, nafill(c(NA, NA, 3, NA, NA), type="locf", limit=1), c(NA, NA, 3, 3, NA))
11 changes: 9 additions & 2 deletions man/nafill.Rd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,16 @@
Fast fill missing values using constant value, \emph{last observation carried forward} or \emph{next observation carried backward}.
}
\usage{
nafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA)
setnafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x))
nafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, limit=Inf)
setnafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x), limit=Inf)
}
\arguments{
\item{x}{ Vector, list, data.frame or data.table of logical, numeric or character columns. }
\item{type}{ Character, one of \emph{"const"}, \emph{"locf"} or \emph{"nocb"}. Defaults to \code{"const"}. }
\item{fill}{ Value to be used to replace missing observations. See examples. }
\item{nan}{ Either \code{NaN} or \code{NA}; if the former, \code{NaN} is treated as distinct from \code{NA}, otherwise, they are treated the same during replacement. See Examples. }
\item{cols}{ Numeric or character vector specifying columns to be updated. }
\item{limit}{ The maximum number of consecutive \code{NA} values to fill. Must be a non-negative scalar numeric. Default is \code{Inf}. }
}
\details{
Supported types are \emph{logical}, \emph{integer}, \emph{double}, \emph{character}, and \emph{factor}, as well as classes built on top of these such as \code{Date}, \code{IDate}, and \code{POSIXct}.
Expand Down Expand Up @@ -51,6 +52,12 @@ nafill(dt, "nocb")

setnafill(dt, "locf", cols=c("v2","v3"))
dt

# limit= restricts the number of consecutive fills
y = c(1, NA, NA, NA, 5)
nafill(y, "locf", limit=1) # Only fills the first NA
nafill(y, "locf", limit=2) # Fills the first two NAs

}
\seealso{
\code{\link{shift}}, \code{\link{data.table}}, \code{\link{fcoalesce}}
Expand Down
8 changes: 5 additions & 3 deletions src/data.table.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,9 +311,11 @@ SEXP copyAsGrowable(SEXP x);
SEXP resizeVector(SEXP x, SEXP size);

// nafill.c
void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose);
void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose);
SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols);
void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose, double limit);
void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose, double limit);
void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fill, ans_t *ans, bool verbose, double limit);
void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, ans_t *ans, bool verbose, double limit);
SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols, SEXP limit);

// between.c
SEXP between(SEXP x, SEXP lower, SEXP upper, SEXP incbounds, SEXP NAbounds, SEXP check);
Expand Down
88 changes: 69 additions & 19 deletions src/nafill.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#include "data.table.h"

void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose) {
void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose, double limit) {
double tic=0.0;
if (verbose)
tic = omp_get_wtime();
Expand All @@ -15,34 +15,52 @@ void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, b
}
}
} else if (type==1) { // locf
uint_fast64_t fills = 0;
if (nan_is_na) {
ans->dbl_v[0] = ISNAN(x[0]) ? fill : x[0];
if (ISNAN(x[0])) fills = 1;
for (uint_fast64_t i=1; i<nx; i++) {
ans->dbl_v[i] = ISNAN(x[i]) ? ans->dbl_v[i-1] : x[i];
if (ISNAN(x[i])) {
if (fills < limit) { ans->dbl_v[i] = ans->dbl_v[i-1]; fills++; }
else ans->dbl_v[i] = x[i];
} else { ans->dbl_v[i] = x[i]; fills = 0; }
}
} else {
ans->dbl_v[0] = ISNA(x[0]) ? fill : x[0];
if (ISNA(x[0])) fills = 1;
for (uint_fast64_t i=1; i<nx; i++) {
ans->dbl_v[i] = ISNA(x[i]) ? ans->dbl_v[i-1] : x[i];
if (ISNA(x[i])) {
if (fills < limit) { ans->dbl_v[i] = ans->dbl_v[i-1]; fills++; }
else ans->dbl_v[i] = x[i];
} else { ans->dbl_v[i] = x[i]; fills = 0; }
}
}
} else if (type==2) { // nocb
uint_fast64_t fills = 0;
if (nan_is_na) {
ans->dbl_v[nx-1] = ISNAN(x[nx-1]) ? fill : x[nx-1];
if (ISNAN(x[nx-1])) fills = 1;
for (int_fast64_t i=nx-2; i>=0; i--) {
ans->dbl_v[i] = ISNAN(x[i]) ? ans->dbl_v[i+1] : x[i];
if (ISNAN(x[i])) {
if (fills < limit) { ans->dbl_v[i] = ans->dbl_v[i+1]; fills++; }
else ans->dbl_v[i] = x[i];
} else { ans->dbl_v[i] = x[i]; fills = 0; }
}
} else {
ans->dbl_v[nx-1] = ISNA(x[nx-1]) ? fill : x[nx-1];
if (ISNA(x[nx-1])) fills = 1;
for (int_fast64_t i=nx-2; i>=0; i--) {
ans->dbl_v[i] = ISNA(x[i]) ? ans->dbl_v[i+1] : x[i];
if (ISNA(x[i])) {
if (fills < limit) { ans->dbl_v[i] = ans->dbl_v[i+1]; fills++; }
else ans->dbl_v[i] = x[i];
} else { ans->dbl_v[i] = x[i]; fills = 0; }
}
}
}
if (verbose)
snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic);
}
void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose) {
void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose, double limit) {
double tic=0.0;
if (verbose)
tic = omp_get_wtime();
Expand All @@ -51,20 +69,30 @@ void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill
ans->int_v[i] = x[i]==NA_INTEGER ? fill : x[i];
}
} else if (type==1) { // locf
uint_fast64_t fills = 0;
ans->int_v[0] = x[0]==NA_INTEGER ? fill : x[0];
if (x[0]==NA_INTEGER) fills = 1;
for (uint_fast64_t i=1; i<nx; i++) {
ans->int_v[i] = x[i]==NA_INTEGER ? ans->int_v[i-1] : x[i];
if (x[i]==NA_INTEGER) {
if (fills < limit) { ans->int_v[i] = ans->int_v[i-1]; fills++; }
else ans->int_v[i] = x[i];
} else { ans->int_v[i] = x[i]; fills = 0; }
}
} else if (type==2) { // nocb
uint_fast64_t fills = 0;
ans->int_v[nx-1] = x[nx-1]==NA_INTEGER ? fill : x[nx-1];
if (x[nx-1]==NA_INTEGER) fills = 1;
for (int_fast64_t i=nx-2; i>=0; i--) {
ans->int_v[i] = x[i]==NA_INTEGER ? ans->int_v[i+1] : x[i];
if (x[i]==NA_INTEGER) {
if (fills < limit) { ans->int_v[i] = ans->int_v[i+1]; fills++; }
else ans->int_v[i] = x[i];
} else { ans->int_v[i] = x[i]; fills = 0; }
}
}
if (verbose)
snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic);
}
void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fill, ans_t *ans, bool verbose) {
void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fill, ans_t *ans, bool verbose, double limit) {
double tic=0.0;
if (verbose)
tic = omp_get_wtime();
Expand All @@ -73,21 +101,31 @@ void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fi
ans->int64_v[i] = x[i]==NA_INTEGER64 ? fill : x[i];
}
} else if (type==1) { // locf
uint_fast64_t fills = 0;
ans->int64_v[0] = x[0]==NA_INTEGER64 ? fill : x[0];
if (x[0]==NA_INTEGER64) fills = 1;
for (uint_fast64_t i=1; i<nx; i++) {
ans->int64_v[i] = x[i]==NA_INTEGER64 ? ans->int64_v[i-1] : x[i];
if (x[i]==NA_INTEGER64) {
if (fills < limit) { ans->int64_v[i] = ans->int64_v[i-1]; fills++; }
else ans->int64_v[i] = x[i];
} else { ans->int64_v[i] = x[i]; fills = 0; }
}
} else if (type==2) { // nocb
uint_fast64_t fills = 0;
ans->int64_v[nx-1] = x[nx-1]==NA_INTEGER64 ? fill : x[nx-1];
if (x[nx-1]==NA_INTEGER64) fills = 1;
for (int_fast64_t i=nx-2; i>=0; i--) {
ans->int64_v[i] = x[i]==NA_INTEGER64 ? ans->int64_v[i+1] : x[i];
if (x[i]==NA_INTEGER64) {
if (fills < limit) { ans->int64_v[i] = ans->int64_v[i+1]; fills++; }
else ans->int64_v[i] = x[i];
} else { ans->int64_v[i] = x[i]; fills = 0; }
}
}
if (verbose)
snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic);
}

void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, ans_t *ans, bool verbose) {
void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, ans_t *ans, bool verbose, double limit) {
double tic=0.0;
if (verbose)
tic = omp_get_wtime();
Expand All @@ -96,16 +134,26 @@ void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill,
SET_STRING_ELT(ans->char_v, i, x[i]==NA_STRING ? fill : x[i]);
}
} else if (type==1) { // locf
uint_fast64_t fills = 0;
SET_STRING_ELT(ans->char_v, 0, x[0]==NA_STRING ? fill : x[0]);
if (x[0]==NA_STRING) fills = 1;
const SEXP* thisans = SEXPPTR_RO(ans->char_v); // takes out STRING_ELT from loop
for (uint_fast64_t i=1; i<nx; i++) {
SET_STRING_ELT(ans->char_v, i, x[i]==NA_STRING ? thisans[i-1] : x[i]);
if (x[i]==NA_STRING) {
if (fills < limit) { SET_STRING_ELT(ans->char_v, i, thisans[i-1]); fills++; }
else SET_STRING_ELT(ans->char_v, i, x[i]);
} else { SET_STRING_ELT(ans->char_v, i, x[i]); fills = 0; }
}
} else if (type==2) { // nocb
uint_fast64_t fills = 0;
SET_STRING_ELT(ans->char_v, nx-1, x[nx-1]==NA_STRING ? fill : x[nx-1]);
if (x[nx-1]==NA_STRING) fills = 1;
const SEXP* thisans = SEXPPTR_RO(ans->char_v); // takes out STRING_ELT from loop
for (int_fast64_t i=nx-2; i>=0; i--) {
SET_STRING_ELT(ans->char_v, i, x[i]==NA_STRING ? thisans[i+1] : x[i]);
if (x[i]==NA_STRING) {
if (fills < limit) { SET_STRING_ELT(ans->char_v, i, thisans[i+1]); fills++; }
else SET_STRING_ELT(ans->char_v, i, x[i]);
} else { SET_STRING_ELT(ans->char_v, i, x[i]); fills = 0; }
}
}
if (verbose)
Expand All @@ -117,7 +165,7 @@ void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill,
over columns of the input data. This includes handling different data types
and applying the designated filling method to each column in parallel.
*/
SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols) {
SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols, SEXP limit) {
int protecti=0;
const bool verbose = GetVerbose();

Expand All @@ -128,6 +176,8 @@ SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, S
if (verbose)
tic = omp_get_wtime();

double limit_val = REAL(limit)[0];

bool copy = !LOGICAL(inplace)[0];
if (!IS_TRUE_OR_FALSE(nan_is_na_arg))
error(_("'%s' must be TRUE or FALSE"), "nan_is_na"); // # nocov
Expand Down Expand Up @@ -253,16 +303,16 @@ SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, S
switch (TYPEOF(VECTOR_ELT(x, i))) {
case REALSXP : {
if (isInt64[i]) {
nafillInteger64(i64x[i], inx[i], itype, hasFill ? ((int64_t *)fillp[i])[0] : NA_INTEGER64, &vans[i], verbose);
nafillInteger64(i64x[i], inx[i], itype, hasFill ? ((int64_t *)fillp[i])[0] : NA_INTEGER64, &vans[i], verbose, limit_val);
} else {
nafillDouble(dx[i], inx[i], itype, hasFill ? ((double *)fillp[i])[0] : NA_REAL, nan_is_na, &vans[i], verbose);
nafillDouble(dx[i], inx[i], itype, hasFill ? ((double *)fillp[i])[0] : NA_REAL, nan_is_na, &vans[i], verbose, limit_val);
}
} break;
case LGLSXP: case INTSXP : {
nafillInteger(ix[i], inx[i], itype, hasFill ? ((int32_t *)fillp[i])[0] : NA_INTEGER, &vans[i], verbose);
nafillInteger(ix[i], inx[i], itype, hasFill ? ((int32_t *)fillp[i])[0] : NA_INTEGER, &vans[i], verbose, limit_val);
} break;
case STRSXP : {
nafillString(sx[i], inx[i], itype, hasFill ? ((SEXP *)fillp[i])[0] : NA_STRING, &vans[i], verbose);
nafillString(sx[i], inx[i], itype, hasFill ? ((SEXP *)fillp[i])[0] : NA_STRING, &vans[i], verbose, limit_val);
} break;
}
}
Expand Down
Loading