diff --git a/cwms-data-api/src/main/java/cwms/cda/api/CatalogController.java b/cwms-data-api/src/main/java/cwms/cda/api/CatalogController.java index a4d04a7e4..3be9574ac 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/CatalogController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/CatalogController.java @@ -42,6 +42,7 @@ public class CatalogController implements CrudHandler { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); private static final String TAG = "Catalog"; public static final boolean INCLUDE_EXTENTS_DEFAULT = true; + public static final boolean INCLUDE_VERSIONS_DEFAULT = true; public static final boolean EXCLUDE_EMPTY_DEFAULT = true; private final MetricRegistry metrics; @@ -133,6 +134,13 @@ public void getAll(Context ctx) { + "extents. Only valid for TIMESERIES. Note: This parameter is " + "unsupported when dataset is Locations." + "Default is " + INCLUDE_EXTENTS_DEFAULT + "."), + @OpenApiParam(name = INCLUDE_VERSIONS, type = Boolean.class, + description = "Whether the returned catalog entries should include timeseries " + + "versions in the extents block. " + + "Only used when include-extents is enabled, otherwise it is ignored. " + + "Only valid for TIMESERIES. Note: This parameter is " + + "unsupported when dataset is Locations." + + "Default is " + INCLUDE_VERSIONS_DEFAULT + "."), @OpenApiParam(name = EXCLUDE_EMPTY, type = Boolean.class, description = "Specifies " + "whether Timeseries that have empty extents " @@ -245,6 +253,8 @@ public void getOne(@NotNull Context ctx, @NotNull String dataSet) { boolean includeExtents = ctx.queryParamAsClass(INCLUDE_EXTENTS, Boolean.class) .getOrDefault(INCLUDE_EXTENTS_DEFAULT); + boolean includeVersions = ctx.queryParamAsClass(INCLUDE_VERSIONS, Boolean.class) + .getOrDefault(INCLUDE_VERSIONS_DEFAULT); boolean excludeExtents = ctx.queryParamAsClass(EXCLUDE_EMPTY, Boolean.class) .getOrDefault(EXCLUDE_EMPTY_DEFAULT); @@ -257,6 +267,7 @@ public void getOne(@NotNull Context ctx, @NotNull String dataSet) { .withTsGroupLike(tsGroupLike) .withBoundingOfficeLike(boundingOfficeLike) .withIncludeExtents(includeExtents) + .withIncludeVersions(includeVersions) .withExcludeEmpty(excludeExtents) .withLocationKind(locationKind) .withLocationType(locationType) @@ -268,7 +279,7 @@ public void getOne(@NotNull Context ctx, @NotNull String dataSet) { } else if (LOCATIONS.equalsIgnoreCase(valDataSet)) { warnAboutNotSupported(ctx, new String[]{TIMESERIES_CATEGORY_LIKE, - TIMESERIES_GROUP_LIKE, EXCLUDE_EMPTY, INCLUDE_EXTENTS}); + TIMESERIES_GROUP_LIKE, EXCLUDE_EMPTY, INCLUDE_EXTENTS, INCLUDE_VERSIONS}); CatalogRequestParameters parameters = new CatalogRequestParameters.Builder() .withUnitSystem(unitSystem) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java b/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java index e4059e4e1..04bcf6eb8 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java @@ -217,6 +217,7 @@ public final class Controllers { public static final String DESIGNATOR = "designator"; public static final String DESIGNATOR_MASK = "designator-mask"; public static final String INCLUDE_EXTENTS = "include-extents"; + public static final String INCLUDE_VERSIONS = "include-versions"; public static final String EXCLUDE_EMPTY = "exclude-empty"; public static final String DEFAULT_VALUE = "default-value"; public static final String CATEGORY = "category"; diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/CatalogRequestParameters.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/CatalogRequestParameters.java index e031e8f50..cdffb65d9 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/CatalogRequestParameters.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/CatalogRequestParameters.java @@ -17,6 +17,7 @@ public class CatalogRequestParameters { private final String tsGroupLike; private final String boundingOfficeLike; private final boolean includeExtents; + private final boolean includeVersions; private final boolean excludeEmpty; private final String locationKind; private final String locationType; @@ -34,6 +35,7 @@ private CatalogRequestParameters(Builder builder) { this.tsGroupLike = builder.tsGroupLike; this.boundingOfficeLike = builder.boundingOfficeLike; this.includeExtents = builder.includeExtents; + this.includeVersions = builder.includeVersions; this.excludeEmpty = builder.excludeEmpty; this.locationKind = builder.locationKind; this.locationType = builder.locationType; @@ -54,6 +56,10 @@ public boolean isIncludeExtents() { return includeExtents; } + public boolean isIncludeVersions() { + return includeVersions; + } + public String getLocCatLike() { return locCatLike; } @@ -112,6 +118,7 @@ public static class Builder { String tsGroupLike; String boundingOfficeLike; boolean includeExtents = false; + boolean includeVersions = false; private boolean excludeEmpty = true; String locationKind; String locationType; @@ -168,6 +175,11 @@ public Builder withIncludeExtents(boolean includeExtents) { return this; } + public Builder withIncludeVersions(boolean includeVersions) { + this.includeVersions = includeVersions; + return this; + } + public Builder withExcludeEmpty(boolean excludeExtents) { this.excludeEmpty = excludeExtents; return this; @@ -210,6 +222,7 @@ public static Builder from(CatalogRequestParameters params) { .withTsGroupLike(params.tsGroupLike) .withBoundingOfficeLike(params.boundingOfficeLike) .withIncludeExtents(params.includeExtents) + .withIncludeVersions(params.includeVersions) .withExcludeEmpty(params.excludeEmpty) .withLocationKind(params.locationKind) .withLocationType(params.locationType) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 6dabcad23..578cc21b9 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -21,8 +21,10 @@ import static org.jooq.impl.DSL.select; import static org.jooq.impl.DSL.selectDistinct; +import org.jooq.SelectOnConditionStep; import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; import static org.jooq.impl.DSL.table; +import static usace.cwms.db.jooq.codegen.tables.AT_CWMS_TS_SPEC.AT_CWMS_TS_SPEC; import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; import static usace.cwms.db.jooq.codegen.tables.AV_TS_EXTENTS_UTC.AV_TS_EXTENTS_UTC; @@ -676,6 +678,7 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar .withTsGroupLike(catPage.getTsGroupLike()) .withBoundingOfficeLike(catPage.getBoundingOfficeLike()) .withIncludeExtents(catPage.isIncludeExtents()) + .withIncludeExtents(catPage.isIncludeVersions()) .withExcludeEmpty(catPage.isExcludeEmpty()) .build(); } @@ -690,10 +693,11 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar List pagingConditions = buildPagingConditions(cwmsTsIdFields, cursorOffice, cursorTsId); CommonTableExpression limiter = buildWithClause(cwmsTsIdFields, params, whereConditions, pagingConditions, pageSize, false); Field limiterCode = limiter.field(cwmsTsIdFields.getTsCode()); - SelectJoinStep tmpQuery = dsl.with(limiter) + SelectOnConditionStep tmpQuery = dsl.with(limiter) .select(pageEntryFields) .from(limiter) - .join(table).on(limiterCode.eq(cwmsTsIdFields.getTsCode())); + .join(table).on(limiterCode.eq(cwmsTsIdFields.getTsCode())) + .leftJoin(AT_CWMS_TS_SPEC).on(limiterCode.eq(AT_CWMS_TS_SPEC.TS_CODE)); if (params.isIncludeExtents()) { @@ -701,7 +705,11 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar .on(limiterCode .eq(AV_TS_EXTENTS_UTC.TS_CODE.coerce(limiterCode))); } - final SelectSeekStep2 overallQuery = tmpQuery + Condition versionsCondition = noCondition(); + if (params.isIncludeExtents() && !params.isIncludeVersions()) { + versionsCondition = versionsCondition.and(AV_TS_EXTENTS_UTC.VERSION_TIME.isNull()); + } + final SelectSeekStep2 overallQuery = tmpQuery.where(versionsCondition) .orderBy(cwmsTsIdFields.getDbOfficeId(), cwmsTsIdFields.getCwmsTsId()); logger.atFine().log("%s", lazy(() -> overallQuery.getSQL(ParamType.INLINED))); @@ -721,7 +729,8 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar .cwmsTsId(row.get(cwmsTsIdFields.getCwmsTsId())) .units(row.get(cwmsTsIdFields.getUnitId())) .interval(row.get(cwmsTsIdFields.getIntervalId())) - .intervalOffset(row.get(cwmsTsIdFields.getIntervalUtcOffset())); + .intervalOffset(row.get(cwmsTsIdFields.getIntervalUtcOffset())) + .versioned(parseBool(row.get(AT_CWMS_TS_SPEC.VERSION_FLAG))); builder.timeZone(row.get("TIME_ZONE_ID", String.class)); @@ -849,6 +858,7 @@ private void updateAliasMapping(Map> tsCodeAliasMap retVal.add(cwmsTsIdFields.getIntervalId()); retVal.add(cwmsTsIdFields.getIntervalUtcOffset()); retVal.add(cwmsTsIdFields.getTimeZoneId()); + retVal.add(AT_CWMS_TS_SPEC.VERSION_FLAG); if(cwmsTsIdFields.includesAliases()) { retVal.add(AV_CWMS_TS_ID2.ALIASED_ITEM); retVal.add(AV_CWMS_TS_ID2.TS_CODE); @@ -985,7 +995,6 @@ private Collection buildExtentsConditions(CatalogRequestPar AV_TS_EXTENTS_UTC.LAST_UPDATE.isNotNull()) ); } - return retval; } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/Catalog.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/Catalog.java index 0f2fc6ab6..6f1f6ad8b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/Catalog.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/Catalog.java @@ -76,6 +76,7 @@ public static class CatalogPage { private final String tsGroupLike; private final String boundingOfficeLike; private final boolean includeExtents; + private final boolean includeVersions; private final boolean excludeEmpty; private int total; private int pageSize; @@ -83,7 +84,7 @@ public static class CatalogPage { public CatalogPage(String page) { String[] parts = CwmsDTOPaginated.decodeCursor(page, CwmsDTOPaginated.delimiter); - if (parts.length != 12) { + if (parts.length != 13) { throw new IllegalArgumentException("Invalid Catalog Page Provided, please verify " + "you are using a page variable from the catalog endpoint"); } @@ -98,9 +99,10 @@ public CatalogPage(String page) { tsGroupLike = nullOrVal(parts[6]); boundingOfficeLike = nullOrVal(parts[7]); includeExtents = Boolean.parseBoolean(parts[8]); - excludeEmpty = Boolean.parseBoolean(parts[9]); - total = Integer.parseInt(parts[10]); - pageSize = Integer.parseInt(parts[11]); + includeVersions = Boolean.parseBoolean(parts[9]); + excludeEmpty = Boolean.parseBoolean(parts[10]); + total = Integer.parseInt(parts[11]); + pageSize = Integer.parseInt(parts[12]); } @@ -118,6 +120,7 @@ public CatalogPage(String curElement, CatalogRequestParameters params) { this.tsGroupLike = params.getTsGroupLike(); this.boundingOfficeLike = params.getBoundingOfficeLike(); this.includeExtents = params.isIncludeExtents(); + this.includeVersions = params.isIncludeVersions(); this.excludeEmpty = params.isExcludeEmpty(); } @@ -177,6 +180,10 @@ public boolean isIncludeExtents() { return includeExtents; } + public boolean isIncludeVersions() { + return includeVersions; + } + public boolean isExcludeEmpty() { return excludeEmpty; } @@ -193,6 +200,7 @@ public String toString() { + CwmsDTOPaginated.delimiter + tsGroupLike + CwmsDTOPaginated.delimiter + boundingOfficeLike + CwmsDTOPaginated.delimiter + includeExtents + + CwmsDTOPaginated.delimiter + includeVersions + CwmsDTOPaginated.delimiter + excludeEmpty ; } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/catalog/TimeseriesCatalogEntry.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/catalog/TimeseriesCatalogEntry.java index 44a3c8e61..ad8ef5e3a 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/catalog/TimeseriesCatalogEntry.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/catalog/TimeseriesCatalogEntry.java @@ -34,6 +34,8 @@ public class TimeseriesCatalogEntry extends CatalogEntry { @JacksonXmlProperty(localName = "alias") private Collection aliases; + private boolean versioned = false; + public String getName() { return this.name; } @@ -60,6 +62,10 @@ public Collection getAliases() { return aliases; } + public boolean isVersioned() { + return versioned; + } + private TimeseriesCatalogEntry() { super(null); } @@ -73,6 +79,7 @@ private TimeseriesCatalogEntry(Builder builder) { this.timeZone = builder.timeZone; this.extents = builder.extents; this.aliases = builder.aliases; + this.versioned = builder.versioned; } public String getUnits() { @@ -100,6 +107,7 @@ public static class Builder { private ZonedDateTime latestTime; private List extents = null; private Collection aliases = null; + private boolean versioned = false; public Builder officeId(final String office) { this.office = office; @@ -162,6 +170,11 @@ public Builder withAliases(final Collection aliases) { return this; } + public Builder versioned(boolean versioned) { + this.versioned = versioned; + return this; + } + public TimeseriesCatalogEntry build() { return new TimeseriesCatalogEntry(this); } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/CatalogControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/CatalogControllerTestIT.java index cc10e68b5..768c2e27b 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/CatalogControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/CatalogControllerTestIT.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import static cwms.cda.api.Controllers.*; import cwms.cda.api.enums.UnitSystem; +import cwms.cda.data.dto.TimeSeries; import cwms.cda.data.dto.basin.Basin; import cwms.cda.data.dto.catalog.TimeSeriesAlias; import cwms.cda.data.dto.catalog.TimeseriesCatalogEntry; @@ -13,6 +14,10 @@ import cwms.cda.formatters.ContentType; import cwms.cda.formatters.json.JsonV2; import fixtures.TestAccounts; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -30,6 +35,7 @@ import java.time.ZoneId; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.io.IOUtils; import org.jooq.DSLContext; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -51,6 +57,7 @@ public class CatalogControllerTestIT extends DataApiTestIT { public static final String OFFICE = "SPK"; + private static TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; //// These have to match the groups in ts_catalog_setup.sql public static final String A_TO_M = "A to M"; @@ -85,6 +92,29 @@ static void setup_data() throws Exception { loadSqlDataFromResource("cwms/cda/data/sql/ts_catalog_setup.sql"); loadSqlDataFromResource("cwms/cda/data/sql/location_catalog_setup.sql"); + + InputStream resource = CatalogController.class.getResourceAsStream("cwms/cda/api/template_num_ts_create.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + assertNotNull(tsData); + tsData = tsData.replace("{OFFICE}", OFFICE) + .replace("{TSID}", "Wet Meadows.Depth-SWE.Inst.15Minutes.0.four") + .replace("{UNITS}", "ft") + .replace("{VERSION_DATE}", Instant.now().truncatedTo(ChronoUnit.MINUTES).toString()); + + given() + .log().ifValidationFails(LogDetail.ALL,true) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); } private static void createProject(String id, String office) throws SQLException { @@ -162,6 +192,26 @@ void test_no_aliases_returned(String format) { assertEquals(0, (int) numAliases, "Expected no aliases, but found some."); } + @ParameterizedTest + @ValueSource(strings = {Formats.JSONV2, Formats.DEFAULT}) + void test_no_versions_returned(String format) { + given() + .accept(format) + .log().ifValidationFails(LogDetail.ALL, true) + .queryParam(Controllers.OFFICE, OFFICE) + .queryParam(EXCLUDE_EMPTY, true) + .queryParam(INCLUDE_VERSIONS, false) + .when() + .get("/catalog/TIMESERIES") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(200)) + .body("entries.findAll { it.versioned == true }.size()", greaterThanOrEqualTo(1)) + .body("entries.extents.flatten()", everyItem(not(hasKey("version-time")))) + ; + } + @ParameterizedTest @ValueSource(strings = {Formats.JSONV2, Formats.DEFAULT}) void test_aliases_returned(String format) { diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/template_num_ts_create.json b/cwms-data-api/src/test/resources/cwms/cda/api/template_num_ts_create.json new file mode 100644 index 000000000..d624d28b6 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/template_num_ts_create.json @@ -0,0 +1,14 @@ +{ + "office-id": "{OFFICE}", + "name": "{TSID}", + "units": "{UNITS}", + "version-date": "{VERSION_DATE}", + "values": [ + [ + 1209654000000, + 4, + 0 + ] + ] +} +