CWMSVUE-755 add "versioned" field to TimeSeriesCatalogEntry

cwms-data-api/src/main/java/cwms/cda/api/CatalogController.java

@@ -42,6 +42,7 @@ public class CatalogController implements CrudHandler {
     private static final FluentLogger logger = FluentLogger.forEnclosingClass();
     private static final String TAG = "Catalog";
     public static final boolean INCLUDE_EXTENTS_DEFAULT = true;
+    public static final boolean INCLUDE_VERSIONS_DEFAULT = true;
     public static final boolean EXCLUDE_EMPTY_DEFAULT = true;
 
     private final MetricRegistry metrics;
@@ -133,6 +134,13 @@ public void getAll(Context ctx) {
                         + "extents. Only valid for TIMESERIES. Note: This parameter is "
                             + "unsupported when dataset is Locations."
                         + "Default is " + INCLUDE_EXTENTS_DEFAULT + "."),
+            @OpenApiParam(name = INCLUDE_VERSIONS, type = Boolean.class,
+                description = "Whether the returned catalog entries should include timeseries "
+                    + "versions in the extents block. "
+                    + "Only used when include-extents is enabled, otherwise it is ignored. "
+                    + "Only valid for TIMESERIES. Note: This parameter is "
+                    + "unsupported when dataset is Locations."
+                    + "Default is " + INCLUDE_VERSIONS_DEFAULT + "."),
             @OpenApiParam(name = EXCLUDE_EMPTY, type = Boolean.class,
                     description = "Specifies "
                         + "whether Timeseries that have empty extents "
@@ -245,6 +253,8 @@ public void getOne(@NotNull Context ctx, @NotNull String dataSet) {
 
                 boolean includeExtents = ctx.queryParamAsClass(INCLUDE_EXTENTS, Boolean.class)
                         .getOrDefault(INCLUDE_EXTENTS_DEFAULT);
+                boolean includeVersions = ctx.queryParamAsClass(INCLUDE_VERSIONS, Boolean.class)
+                    .getOrDefault(INCLUDE_VERSIONS_DEFAULT);
                 boolean excludeExtents = ctx.queryParamAsClass(EXCLUDE_EMPTY, Boolean.class)
                         .getOrDefault(EXCLUDE_EMPTY_DEFAULT);
 
@@ -257,6 +267,7 @@ public void getOne(@NotNull Context ctx, @NotNull String dataSet) {
                         .withTsGroupLike(tsGroupLike)
                         .withBoundingOfficeLike(boundingOfficeLike)
                         .withIncludeExtents(includeExtents)
+                        .withIncludeVersions(includeVersions)
                         .withExcludeEmpty(excludeExtents)
                         .withLocationKind(locationKind)
                         .withLocationType(locationType)
@@ -268,7 +279,7 @@ public void getOne(@NotNull Context ctx, @NotNull String dataSet) {
             } else if (LOCATIONS.equalsIgnoreCase(valDataSet)) {
 
                 warnAboutNotSupported(ctx, new String[]{TIMESERIES_CATEGORY_LIKE,
-                        TIMESERIES_GROUP_LIKE, EXCLUDE_EMPTY, INCLUDE_EXTENTS});
+                        TIMESERIES_GROUP_LIKE, EXCLUDE_EMPTY, INCLUDE_EXTENTS, INCLUDE_VERSIONS});
 
                 CatalogRequestParameters parameters = new CatalogRequestParameters.Builder()
                         .withUnitSystem(unitSystem)

cwms-data-api/src/main/java/cwms/cda/api/Controllers.java

@@ -217,6 +217,7 @@ public final class Controllers {
     public static final String DESIGNATOR = "designator";
     public static final String DESIGNATOR_MASK = "designator-mask";
     public static final String INCLUDE_EXTENTS = "include-extents";
+    public static final String INCLUDE_VERSIONS = "include-versions";
     public static final String EXCLUDE_EMPTY = "exclude-empty";
     public static final String DEFAULT_VALUE = "default-value";
     public static final String CATEGORY = "category";

cwms-data-api/src/main/java/cwms/cda/data/dao/CatalogRequestParameters.java

@@ -17,6 +17,7 @@ public class CatalogRequestParameters {
     private final String tsGroupLike;
     private final String boundingOfficeLike;
     private final boolean includeExtents;
+    private final boolean includeVersions;
     private final boolean excludeEmpty;
     private final String locationKind;
     private final String locationType;
@@ -34,6 +35,7 @@ private CatalogRequestParameters(Builder builder) {
         this.tsGroupLike = builder.tsGroupLike;
         this.boundingOfficeLike = builder.boundingOfficeLike;
         this.includeExtents = builder.includeExtents;
+        this.includeVersions = builder.includeVersions;
         this.excludeEmpty = builder.excludeEmpty;
         this.locationKind = builder.locationKind;
         this.locationType = builder.locationType;
@@ -54,6 +56,10 @@ public boolean isIncludeExtents() {
         return includeExtents;
     }
 
+    public boolean isIncludeVersions() {
+        return includeVersions;
+    }
+
     public String getLocCatLike() {
         return locCatLike;
     }
@@ -112,6 +118,7 @@ public static class Builder {
         String tsGroupLike;
         String boundingOfficeLike;
         boolean includeExtents = false;
+        boolean includeVersions = false;
         private boolean excludeEmpty = true;
         String locationKind;
         String locationType;
@@ -168,6 +175,11 @@ public Builder withIncludeExtents(boolean includeExtents) {
             return this;
         }
 
+        public Builder withIncludeVersions(boolean includeVersions) {
+            this.includeVersions = includeVersions;
+            return this;
+        }
+
         public Builder withExcludeEmpty(boolean excludeExtents) {
             this.excludeEmpty = excludeExtents;
             return this;
@@ -210,6 +222,7 @@ public static Builder from(CatalogRequestParameters params) {
                     .withTsGroupLike(params.tsGroupLike)
                     .withBoundingOfficeLike(params.boundingOfficeLike)
                     .withIncludeExtents(params.includeExtents)
+                    .withIncludeVersions(params.includeVersions)
                     .withExcludeEmpty(params.excludeEmpty)
                     .withLocationKind(params.locationKind)
                     .withLocationType(params.locationType)

cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java

@@ -21,8 +21,10 @@
 import static org.jooq.impl.DSL.select;
 import static org.jooq.impl.DSL.selectDistinct;
 
+import org.jooq.SelectOnConditionStep;
 import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID;
 import static org.jooq.impl.DSL.table;
+import static usace.cwms.db.jooq.codegen.tables.AT_CWMS_TS_SPEC.AT_CWMS_TS_SPEC;
 import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2;
 import static usace.cwms.db.jooq.codegen.tables.AV_TS_EXTENTS_UTC.AV_TS_EXTENTS_UTC;
 
@@ -676,6 +678,7 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar
                     .withTsGroupLike(catPage.getTsGroupLike())
                     .withBoundingOfficeLike(catPage.getBoundingOfficeLike())
                     .withIncludeExtents(catPage.isIncludeExtents())
+                    .withIncludeExtents(catPage.isIncludeVersions())
                     .withExcludeEmpty(catPage.isExcludeEmpty())
                     .build();
         }
@@ -690,18 +693,23 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar
         List<Condition> pagingConditions = buildPagingConditions(cwmsTsIdFields, cursorOffice, cursorTsId);
         CommonTableExpression<?> limiter = buildWithClause(cwmsTsIdFields, params, whereConditions, pagingConditions, pageSize, false);
         Field<BigDecimal> limiterCode = limiter.field(cwmsTsIdFields.getTsCode());
-        SelectJoinStep<?> tmpQuery = dsl.with(limiter)
+        SelectOnConditionStep<?> tmpQuery = dsl.with(limiter)
                                         .select(pageEntryFields)
                                         .from(limiter)
-                                        .join(table).on(limiterCode.eq(cwmsTsIdFields.getTsCode()));
+                                        .join(table).on(limiterCode.eq(cwmsTsIdFields.getTsCode()))
+                                        .leftJoin(AT_CWMS_TS_SPEC).on(limiterCode.eq(AT_CWMS_TS_SPEC.TS_CODE));
 
         if (params.isIncludeExtents()) {
 
             tmpQuery = tmpQuery.leftOuterJoin(AV_TS_EXTENTS_UTC)
                                        .on(limiterCode
                                          .eq(AV_TS_EXTENTS_UTC.TS_CODE.coerce(limiterCode)));
         }
-        final SelectSeekStep2<?, String, String> overallQuery = tmpQuery
+        Condition versionsCondition = noCondition();
+        if (params.isIncludeExtents() && !params.isIncludeVersions()) {
+            versionsCondition = versionsCondition.and(AV_TS_EXTENTS_UTC.VERSION_TIME.isNull());
+        }
+        final SelectSeekStep2<?, String, String> overallQuery = tmpQuery.where(versionsCondition)
                 .orderBy(cwmsTsIdFields.getDbOfficeId(),
                         cwmsTsIdFields.getCwmsTsId());
         logger.atFine().log("%s", lazy(() -> overallQuery.getSQL(ParamType.INLINED)));
@@ -721,7 +729,8 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar
                         .cwmsTsId(row.get(cwmsTsIdFields.getCwmsTsId()))
                         .units(row.get(cwmsTsIdFields.getUnitId()))
                         .interval(row.get(cwmsTsIdFields.getIntervalId()))
-                        .intervalOffset(row.get(cwmsTsIdFields.getIntervalUtcOffset()));
+                        .intervalOffset(row.get(cwmsTsIdFields.getIntervalUtcOffset()))
+                        .versioned(parseBool(row.get(AT_CWMS_TS_SPEC.VERSION_FLAG)));
 
                 builder.timeZone(row.get("TIME_ZONE_ID", String.class));
 
@@ -849,6 +858,7 @@ private void updateAliasMapping(Map<String, Set<TimeSeriesAlias>> tsCodeAliasMap
         retVal.add(cwmsTsIdFields.getIntervalId());
         retVal.add(cwmsTsIdFields.getIntervalUtcOffset());
         retVal.add(cwmsTsIdFields.getTimeZoneId());
+        retVal.add(AT_CWMS_TS_SPEC.VERSION_FLAG);
         if(cwmsTsIdFields.includesAliases()) {
             retVal.add(AV_CWMS_TS_ID2.ALIASED_ITEM);
             retVal.add(AV_CWMS_TS_ID2.TS_CODE);
@@ -985,7 +995,6 @@ private Collection<? extends Condition> buildExtentsConditions(CatalogRequestPar
                 AV_TS_EXTENTS_UTC.LAST_UPDATE.isNotNull())
             );
         }
-
         return retval;
     }
 

cwms-data-api/src/main/java/cwms/cda/data/dto/Catalog.java

@@ -76,14 +76,15 @@ public static class CatalogPage {
         private final String tsGroupLike;
         private final String boundingOfficeLike;
         private final boolean includeExtents;
+        private final boolean includeVersions;
         private final boolean excludeEmpty;
         private int total;
         private int pageSize;
 
         public CatalogPage(String page) {
             String[] parts = CwmsDTOPaginated.decodeCursor(page, CwmsDTOPaginated.delimiter);
 
-            if (parts.length != 12) {
+            if (parts.length != 13) {
                 throw new IllegalArgumentException("Invalid Catalog Page Provided, please verify "
                         + "you are using a page variable from the catalog endpoint");
             }
@@ -98,9 +99,10 @@ public CatalogPage(String page) {
             tsGroupLike = nullOrVal(parts[6]);
             boundingOfficeLike = nullOrVal(parts[7]);
             includeExtents = Boolean.parseBoolean(parts[8]);
-            excludeEmpty = Boolean.parseBoolean(parts[9]);
-            total = Integer.parseInt(parts[10]);
-            pageSize = Integer.parseInt(parts[11]);
+            includeVersions = Boolean.parseBoolean(parts[9]);
+            excludeEmpty = Boolean.parseBoolean(parts[10]);
+            total = Integer.parseInt(parts[11]);
+            pageSize = Integer.parseInt(parts[12]);
         }
 
 
@@ -118,6 +120,7 @@ public CatalogPage(String curElement, CatalogRequestParameters params) {
             this.tsGroupLike = params.getTsGroupLike();
             this.boundingOfficeLike = params.getBoundingOfficeLike();
             this.includeExtents = params.isIncludeExtents();
+            this.includeVersions = params.isIncludeVersions();
             this.excludeEmpty = params.isExcludeEmpty();
         }
 
@@ -177,6 +180,10 @@ public boolean isIncludeExtents() {
             return includeExtents;
         }
 
+        public boolean isIncludeVersions() {
+            return includeVersions;
+        }
+
         public boolean isExcludeEmpty() {
             return excludeEmpty;
         }
@@ -193,6 +200,7 @@ public String toString() {
                     + CwmsDTOPaginated.delimiter + tsGroupLike
                     + CwmsDTOPaginated.delimiter + boundingOfficeLike
                     + CwmsDTOPaginated.delimiter + includeExtents
+                    + CwmsDTOPaginated.delimiter + includeVersions
                     + CwmsDTOPaginated.delimiter + excludeEmpty
                     ;
         }

cwms-data-api/src/main/java/cwms/cda/data/dto/catalog/TimeseriesCatalogEntry.java

@@ -34,6 +34,8 @@ public class TimeseriesCatalogEntry extends CatalogEntry {
     @JacksonXmlProperty(localName = "alias")
     private Collection<TimeSeriesAlias> aliases;
 
+    private boolean versioned = false;
+
     public String getName() {
         return this.name;
     }
@@ -60,6 +62,10 @@ public Collection<TimeSeriesAlias> getAliases() {
         return aliases;
     }
 
+    public boolean isVersioned() {
+        return versioned;
+    }
+
     private TimeseriesCatalogEntry() {
         super(null);
     }
@@ -73,6 +79,7 @@ private TimeseriesCatalogEntry(Builder builder) {
         this.timeZone = builder.timeZone;
         this.extents = builder.extents;
         this.aliases = builder.aliases;
+        this.versioned = builder.versioned;
     }
 
     public String getUnits() {
@@ -100,6 +107,7 @@ public static class Builder {
         private ZonedDateTime latestTime;
         private List<TimeSeriesExtents> extents = null;
         private Collection<TimeSeriesAlias> aliases = null;
+        private boolean versioned = false;
 
         public Builder officeId(final String office) {
             this.office = office;
@@ -162,6 +170,11 @@ public Builder withAliases(final Collection<TimeSeriesAlias> aliases) {
             return this;
         }
 
+        public Builder versioned(boolean versioned) {
+            this.versioned = versioned;
+            return this;
+        }
+
         public TimeseriesCatalogEntry build() {
             return new TimeseriesCatalogEntry(this);
         }

cwms-data-api/src/test/java/cwms/cda/api/CatalogControllerTestIT.java

@@ -6,13 +6,18 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import static cwms.cda.api.Controllers.*;
 import cwms.cda.api.enums.UnitSystem;
+import cwms.cda.data.dto.TimeSeries;
 import cwms.cda.data.dto.basin.Basin;
 import cwms.cda.data.dto.catalog.TimeSeriesAlias;
 import cwms.cda.data.dto.catalog.TimeseriesCatalogEntry;
 import cwms.cda.data.dto.stream.Stream;
 import cwms.cda.formatters.ContentType;
 import cwms.cda.formatters.json.JsonV2;
 import fixtures.TestAccounts;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -30,6 +35,7 @@
 
 import java.time.ZoneId;
 import javax.servlet.http.HttpServletResponse;
+import org.apache.commons.io.IOUtils;
 import org.jooq.DSLContext;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
@@ -51,6 +57,7 @@
 public class CatalogControllerTestIT extends DataApiTestIT {
 
     public static final String OFFICE = "SPK";
+    private static TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL;
 
     //// These have to match the groups in ts_catalog_setup.sql
     public static final String A_TO_M = "A to M";
@@ -85,6 +92,29 @@ static void setup_data() throws Exception {
         loadSqlDataFromResource("cwms/cda/data/sql/ts_catalog_setup.sql");
 
         loadSqlDataFromResource("cwms/cda/data/sql/location_catalog_setup.sql");
+
+        InputStream resource = CatalogController.class.getResourceAsStream("cwms/cda/api/template_num_ts_create.json");
+        assertNotNull(resource);
+        String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8);
+        assertNotNull(tsData);
+        tsData = tsData.replace("{OFFICE}", OFFICE)
+            .replace("{TSID}", "Wet Meadows.Depth-SWE.Inst.15Minutes.0.four")
+            .replace("{UNITS}", "ft")
+            .replace("{VERSION_DATE}", Instant.now().truncatedTo(ChronoUnit.MINUTES).toString());
+
+        given()
+            .log().ifValidationFails(LogDetail.ALL,true)
+            .contentType(Formats.JSONV2)
+            .body(tsData)
+            .header("Authorization", user.toHeaderValue())
+        .when()
+            .redirects().follow(true)
+            .redirects().max(3)
+            .post("/timeseries/")
+        .then()
+            .log().ifValidationFails(LogDetail.ALL,true)
+        .assertThat()
+            .statusCode(is(HttpServletResponse.SC_OK));
     }
 
     private static void createProject(String id, String office) throws SQLException {
@@ -162,6 +192,26 @@ void test_no_aliases_returned(String format) {
         assertEquals(0, (int) numAliases, "Expected no aliases, but found some.");
     }
 
+    @ParameterizedTest
+    @ValueSource(strings = {Formats.JSONV2, Formats.DEFAULT})
+    void test_no_versions_returned(String format) {
+        given()
+            .accept(format)
+            .log().ifValidationFails(LogDetail.ALL, true)
+            .queryParam(Controllers.OFFICE, OFFICE)
+            .queryParam(EXCLUDE_EMPTY, true)
+            .queryParam(INCLUDE_VERSIONS, false)
+        .when()
+            .get("/catalog/TIMESERIES")
+        .then()
+            .log().ifValidationFails(LogDetail.ALL, true)
+        .assertThat()
+            .statusCode(is(200))
+            .body("entries.findAll { it.versioned == true }.size()", greaterThanOrEqualTo(1))
+            .body("entries.extents.flatten()", everyItem(not(hasKey("version-time"))))
+            ;
+    }
+
     @ParameterizedTest
     @ValueSource(strings = {Formats.JSONV2, Formats.DEFAULT})
     void test_aliases_returned(String format) {

cwms-data-api/src/test/resources/cwms/cda/api/template_num_ts_create.json

@@ -0,0 +1,14 @@
+{
+  "office-id": "{OFFICE}",
+  "name": "{TSID}",
+  "units": "{UNITS}",
+  "version-date": "{VERSION_DATE}",
+  "values": [
+    [
+      1209654000000,
+      4,
+      0
+    ]
+  ]
+}
+