Add support for GRIB2 DRS 42 (CCSDS compression)

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 1998-2023, University Corporation for Atmospheric Research/Unidata
+Copyright (c) 1998-2025, University Corporation for Atmospheric Research/Unidata
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

build.gradle

@@ -43,7 +43,7 @@ allprojects {
   // We try to follow semantic versioning, and thus we use <major>.<minor>.<patch>-<prerelease version>
   // <prerelease version> may be SNAPSHOT, alphax, betax, etc.
   // Note - if bumping to a new major or minor version, be sure to update the docs
-  version = '5.7.1-SNAPSHOT'
+  version = '5.8.0-SNAPSHOT'
   status = 'development'
 }
 

docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md

@@ -123,13 +123,30 @@ dependencies {
   runtimeOnly "edu.ucar:cdm-mcidas:${netcdfJavaVersion}"
 }
 ~~~
+
+## Native compression code
+
+netCDF-Java `>=v5.8.0` supports libaec compression for GRIB2 messages using JNA to call the libaec C library.
+To ease the use of this feature, we now distribute a jar file containing the native libraries for the following platforms/architectures:
+
+|---
+| platform | x86-64 | aarch64
+|:-|:-:|:-:
+| Linux | <span class="glyphicon glyphicon-ok" style="color: green;"></span> | <span class="glyphicon glyphicon-ok" style="color: green;">
+| MacOS | <span class="glyphicon glyphicon-ok" style="color: green;"> | <span class="glyphicon glyphicon-ok" style="color: green;">
+| Windows | <span class="glyphicon glyphicon-ok" style="color: green;"> | <span class="glyphicon glyphicon-remove" style="color: red;">
+
+If you are using on the of the supported platform/architecture combinations above, you may include the `edu.ucar:libaec-native:${netcdfJavaVersion}` artifact in your project to bypass the need to install libaec on your system.
+Otherwise, libaec will need to be installed and reachable in your system library path in order to read data compressed using libaec.
+
 ## Building with netcdfAll
 
 This is the appropriate option if you’re not using a dependency management tool like Maven or Gradle and you don’t care about jar size or compatibility with other libraries. Simply include netcdfAll-${netcdfJavaVersion}.jar on the classpath when you run your program.
 You’ll also need a logger.
-Currently does not include `cdm-s3` due to the size of the AWS S3 SDK dependency.
+Currently, the netcdfAll jar does not include `cdm-s3` due to the size of the AWS S3 SDK dependency, and does not include the `libaec-native` (native library binaries for libaec).
 
 ## Logging
+
 The netCDF-Java library uses the SLF4J logging facade.
 This allows applications to choose their own logging implementation, by including the appropriate jar file on the classpath at runtime.
 Common choices are `JDK logging` and `Log4J 2`:

grib/build.gradle

@@ -11,6 +11,8 @@ dependencies {
 
   api project(':cdm:cdm-core')
 
+  implementation project(':native-compression:libaec-jna')
+
   implementation 'com.google.protobuf:protobuf-java'
   implementation 'org.jdom:jdom2'
   implementation 'com.google.code.findbugs:jsr305'
@@ -19,11 +21,15 @@ dependencies {
   implementation 'com.beust:jcommander'
   implementation 'com.google.re2j:re2j'
   implementation 'org.slf4j:slf4j-api'
+  implementation 'net.java.dev.jna:jna'
 
   testImplementation project(':cdm-test-utils')
   testImplementation project(':udunits')
 
   testImplementation 'com.google.truth:truth'
   testImplementation 'org.jsoup:jsoup'
+
+  testRuntimeOnly project(':native-compression:libaec-native')
+
   testRuntimeOnly 'ch.qos.logback:logback-classic'
 }

grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java

@@ -1,10 +1,16 @@
 /*
- * Copyright (c) 1998-2021 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
 package ucar.nc2.grib.grib2;
 
+import static edu.ucar.unidata.compression.jna.libaec.LibAec.AEC_OK;
+
+import com.sun.jna.Memory;
+import edu.ucar.unidata.compression.jna.libaec.LibAec;
+import edu.ucar.unidata.compression.jna.libaec.LibAec.AecStream;
+import java.nio.ByteBuffer;
 import javax.annotation.Nullable;
 import ucar.nc2.grib.GribNumbers;
 import ucar.nc2.grib.GribUtils;
@@ -65,18 +71,20 @@ public class Grib2DataReader {
   }
 
   /*
-   * Code Table Code table 5.0 - Data representation template number (5.0)
+   * Code table 5.0 - Data representation template number (5.0)
    * 0: Grid point data - simple packing
    * 1: Matrix value at grid point - simple packing
    * 2: Grid point data - complex packing
    * 3: Grid point data - complex packing and spatial differencing
    * 4: Grid point data - IEEE floating point data
    * 40: Grid point data - JPEG 2000 code stream format
    * 41: Grid point data - Portable Network Graphics (PNG)
+   * 42: Grid point data - CCSDS recommended lossless compression
    * 50: Spectral data - simple packing
    * 51: Spherical harmonics data - complex packing
    * 61: Grid point data - simple packing with logarithm pre-processing
    * 200: Run length packing with level values
+   * 50002: Second order packing
    * 65535: Missing
    */
 
@@ -111,6 +119,9 @@ public float[] getData(RandomAccessFile raf, Grib2SectionBitMap bitmapSection, G
       case 41:
         data = getData41(raf, (Grib2Drs.Type0) gdrs);
         break;
+      case 42:
+        data = getData42(raf, (Grib2Drs.Type42) gdrs);
+        break;
       case 50002:
         data = getData50002(raf, (Grib2Drs.Type50002) gdrs);
         break;
@@ -1073,6 +1084,80 @@ private float[] getData50002(RandomAccessFile raf, Grib2Drs.Type50002 gdrs) thro
 
   }
 
+  private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOException {
+    byte[] decodedData;
+
+    // read CCSDS encoded stream from message
+    int encodedLength = dataLength - 5;
+    byte[] inputData = new byte[encodedLength];
+    raf.readFully(inputData);
+
+    int nbytesPerSample = (gdrs.numberOfBits + 7) / 8;
+
+    try (Memory inputMemory = new Memory(encodedLength);
+        Memory outputMemory = new Memory((long) nbytesPerSample * totalNPoints)) {
+
+      // set encoding parameters
+      AecStream aecStreamDecode = AecStream.create(gdrs.numberOfBits, gdrs.blockSize, gdrs.referenceSampleInterval,
+          gdrs.compressionOptionsMask);
+
+      // load data from grib message into memory
+      inputMemory.write(0, inputData, 0, inputData.length);
+
+      aecStreamDecode.setInputMemory(inputMemory);
+      aecStreamDecode.setOutputMemory(outputMemory);
+
+      // decode
+      int ok = LibAec.aec_buffer_decode(aecStreamDecode);
+      if (ok != AEC_OK) {
+        System.out.printf("AEC Error: %s%n", ok);
+      }
+
+      // read decoded data from native memory
+      decodedData = new byte[nbytesPerSample * totalNPoints];
+      outputMemory.read(0, decodedData, 0, decodedData.length);
+    }
+
+    // will use this to read out a long value using nbytesPerSample bytes
+    // see long getNextLong(ByteBuffer bb, int numberOfBytes)
+    ByteBuffer bb = ByteBuffer.wrap(decodedData);
+
+    // decode following regulation 92.9.4, Note 4
+    int D = gdrs.decimalScaleFactor;
+    float DD = (float) Math.pow((double) 10, (double) D);
+    float R = gdrs.referenceValue;
+    int E = gdrs.binaryScaleFactor;
+    float EE = (float) Math.pow(2.0, (double) E);
+    float[] data = new float[decodedData.length];
+    if (bitmap == null) {
+      for (int i = 0; i < totalNPoints; i++) {
+        data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD;
+      }
+    } else {
+      for (int i = 0; i < totalNPoints; i++) {
+        if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) {
+          data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD;
+        } else {
+          data[i] = staticMissingValue;
+        }
+      }
+    }
+    return data;
+  }
+
+  private long getNextLong(ByteBuffer bb, int numberOfBytes) throws IOException {
+    switch (numberOfBytes) {
+      case 1:
+        return Byte.toUnsignedLong(bb.get());
+      case 2:
+        return Short.toUnsignedLong(bb.getShort());
+      case 4:
+        return Integer.toUnsignedLong(bb.getInt());
+      default:
+        throw new IOException("Invalid number of bytes per sample for GDR42: " + numberOfBytes);
+    }
+  }
+
   /*
    * 
    * Flag table 3.4 – Scanning mode

grib/src/main/java/ucar/nc2/grib/grib2/Grib2Drs.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -22,15 +22,17 @@ public abstract class Grib2Drs {
 
   public static Grib2Drs factory(int template, RandomAccessFile raf) throws IOException {
     switch (template) {
-      case 0:
-      case 41:
+      case 0: // simple packing
+      case 41: // PNG
         return new Type0(raf);
-      case 2:
+      case 2: // complex packing
         return new Type2(raf);
-      case 3:
+      case 3: // complex packing and spatial differencing
         return new Type3(raf);
-      case 40:
+      case 40: // JPEG2000
         return new Type40(raf);
+      case 42: // CCSDS
+        return new Type42(raf);
       case 50002: // ECMWF's second order packing
         return new Type50002(raf);
       default:
@@ -441,6 +443,63 @@ public int hashCode() {
     }
   }
 
+
+  /*
+   * Data representation template 5.42 – Grid point data - CCSDS recommended lossless compression
+   * Note: For most templates, details of the packing process are described in Regulation 92.9.4.
+   * Octet No. Contents
+   * 12-15 Reference value (R) (IEEE 32-bit floating-point value)
+   * 16-17 Binary scale factor (E)
+   * 18-19 Decimal scale factor (D)
+   * 20 Number of bits required to hold the resulting scaled and referenced data values. (see Note 1)
+   * 21 Type of original field values (see Code Table 5.1)
+   * --- additions to Type0 below ---
+   * 22 CCSDS compression options mask (see Note 3)
+   * 23 Block size
+   * 24-25 Reference sample interval
+   */
+  public static class Type42 extends Type0 {
+
+    int compressionOptionsMask, blockSize, referenceSampleInterval;
+
+    Type42(RandomAccessFile raf) throws IOException {
+      super(raf);
+      this.compressionOptionsMask = raf.read();
+      this.blockSize = raf.read();
+      this.referenceSampleInterval = GribNumbers.uint2(raf);
+      System.out.println("hello");
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(this).add("compressionOptionsMask", compressionOptionsMask)
+          .add("blockSize", blockSize).add("referenceSampleInterval", referenceSampleInterval).toString();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (o == null || getClass() != o.getClass()) {
+        return false;
+      }
+      if (!super.equals(o)) {
+        return false;
+      }
+
+      Type42 type2 = (Type42) o;
+
+      if (compressionOptionsMask != type2.compressionOptionsMask) {
+        return false;
+      }
+      if (blockSize != type2.blockSize) {
+        return false;
+      }
+      return referenceSampleInterval == type2.referenceSampleInterval;
+    }
+  }
+
   // pull request #52 "lost-carrier" jkaehler@meteomatics.com
   public static class Type50002 extends Grib2Drs {
 

grib/src/test/java/ucar/nc2/grib/grib2/TestDrs42.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2025 University Corporation for Atmospheric Research/Unidata
+ * See LICENSE for license information.
+ */
+
+package ucar.nc2.grib.grib2;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import java.io.IOException;
+import java.util.Formatter;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import ucar.ma2.Array;
+import ucar.ma2.MAMath;
+import ucar.ma2.MAMath.MinMax;
+import ucar.nc2.NetcdfFile;
+import ucar.nc2.NetcdfFiles;
+import ucar.nc2.Variable;
+import ucar.nc2.util.CompareNetcdf2;
+import ucar.unidata.util.test.TestDir;
+import ucar.unidata.util.test.category.NeedsCdmUnitTest;
+
+public class TestDrs42 {
+
+  @Test
+  public void CCSDS1byte() throws IOException {
+    String origFile = TestDir.localTestDataDir + "sref.pds2.grib2";
+    // drs42File is origFile convert from Simple Packing to CCSDS by ecCodes
+    // grib_set -w isGridded=1 -r -s packingType=grid_ccsds sref.pds2.grib2 sref.pds2.drs42.grib2
+    // This file uses 8 bits per sample in the CCSDS configuration
+    String drs42File = TestDir.localTestDataDir + "sref.pds2.drs42.grib2";
+    final String variableName = "u-component_of_wind_height_above_ground_weightedMean";
+
+    // stats from ecCodes
+    final float expectedMax = 13.4F;
+    final float expectedMin = -9.9F;
+    final float expectedAverage = 0.246189F;
+
+    final int expectedLength = 23865;
+    final float tol = 1e-6F;
+
+    try (NetcdfFile nc42 = NetcdfFiles.open(drs42File)) {
+      Variable v = nc42.findVariable(variableName);
+      assertThat(v != null).isTrue();
+      Array data = v.read();
+
+      assertThat(data).isNotNull();
+      assertThat(data.getSize()).isEqualTo(expectedLength);
+      MinMax extremes = MAMath.getMinMax(data);
+      assertThat(extremes.max).isWithin(tol).of(expectedMax);
+      assertThat(extremes.min).isWithin(tol).of(expectedMin);
+      assertThat(MAMath.sumDouble(data) / data.getSize()).isWithin(tol).of(expectedAverage);
+
+      // compare repacked ccsds data with original data
+      try (NetcdfFile ncOrig = NetcdfFiles.open(origFile)) {
+        Formatter f = new Formatter();
+        CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
+        boolean ok = compare.compare(ncOrig, nc42, null);
+        System.out.printf("%s %s%n", ok ? "OK" : "NOT OK", f);
+        assertThat(ok).isTrue();
+      }
+    }
+  }
+
+  @Test
+  @Category(NeedsCdmUnitTest.class)
+  public void checkVariable2Bytes() throws IOException {
+    // This file uses 16 bits per sample in the CCSDS configuration
+    String drs42File = TestDir.cdmUnitTestDir + "formats/grib2/drs42/"
+        + "icon-eu_europe_regular-lat-lon_single-level_2025031912_014_T_2M_CCSDS.grib2";
+    final String variableName = "Temperature_height_above_ground";
+
+    // stats from ecCodes
+    final float expectedMax = 298.187F;
+    final float expectedMin = 255.639F;
+    final float expectedAverage = 278.009F;
+
+    final int expectedLength = 904689;
+    final float tol = 1e-3F;
+
+    try (NetcdfFile nc42 = NetcdfFiles.open(drs42File)) {
+      Variable v = nc42.findVariable(variableName);
+      assertThat(v != null).isTrue();
+      Array data = v.read();
+
+      assertThat(data).isNotNull();
+      assertThat(data.getSize()).isEqualTo(expectedLength);
+      MinMax extremes = MAMath.getMinMax(data);
+      assertThat(extremes.max).isWithin(tol).of(expectedMax);
+      assertThat(extremes.min).isWithin(tol).of(expectedMin);
+      assertThat(MAMath.sumDouble(data) / data.getSize()).isWithin(tol).of(expectedAverage);
+    }
+  }
+}