diff --git a/LICENSE b/LICENSE index 857e3e1f36..e974842e46 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 1998-2023, University Corporation for Atmospheric Research/Unidata +Copyright (c) 1998-2025, University Corporation for Atmospheric Research/Unidata All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/build.gradle b/build.gradle index 53d732edee..3aba602d43 100644 --- a/build.gradle +++ b/build.gradle @@ -43,7 +43,7 @@ allprojects { // We try to follow semantic versioning, and thus we use ..- // may be SNAPSHOT, alphax, betax, etc. // Note - if bumping to a new major or minor version, be sure to update the docs - version = '5.7.1-SNAPSHOT' + version = '5.8.0-SNAPSHOT' status = 'development' } diff --git a/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md b/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md index 1f356ca547..b1ead0fb64 100644 --- a/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md +++ b/docs/src/site/pages/netcdfJava_tutorial/overview/UsingNetcdfJava.md @@ -123,13 +123,30 @@ dependencies { runtimeOnly "edu.ucar:cdm-mcidas:${netcdfJavaVersion}" } ~~~ + +## Native compression code + +netCDF-Java `>=v5.8.0` supports libaec compression for GRIB2 messages using JNA to call the libaec C library. +To ease the use of this feature, we now distribute a jar file containing the native libraries for the following platforms/architectures: + +|--- +| platform | x86-64 | aarch64 +|:-|:-:|:-: +| Linux | | +| MacOS | | +| Windows | | + +If you are using on the of the supported platform/architecture combinations above, you may include the `edu.ucar:libaec-native:${netcdfJavaVersion}` artifact in your project to bypass the need to install libaec on your system. +Otherwise, libaec will need to be installed and reachable in your system library path in order to read data compressed using libaec. + ## Building with netcdfAll This is the appropriate option if you’re not using a dependency management tool like Maven or Gradle and you don’t care about jar size or compatibility with other libraries. Simply include netcdfAll-${netcdfJavaVersion}.jar on the classpath when you run your program. You’ll also need a logger. -Currently does not include `cdm-s3` due to the size of the AWS S3 SDK dependency. +Currently, the netcdfAll jar does not include `cdm-s3` due to the size of the AWS S3 SDK dependency, and does not include the `libaec-native` (native library binaries for libaec). ## Logging + The netCDF-Java library uses the SLF4J logging facade. This allows applications to choose their own logging implementation, by including the appropriate jar file on the classpath at runtime. Common choices are `JDK logging` and `Log4J 2`: diff --git a/grib/build.gradle b/grib/build.gradle index 047909fd9c..1af91b43ba 100644 --- a/grib/build.gradle +++ b/grib/build.gradle @@ -11,6 +11,8 @@ dependencies { api project(':cdm:cdm-core') + implementation project(':native-compression:libaec-jna') + implementation 'com.google.protobuf:protobuf-java' implementation 'org.jdom:jdom2' implementation 'com.google.code.findbugs:jsr305' @@ -19,11 +21,15 @@ dependencies { implementation 'com.beust:jcommander' implementation 'com.google.re2j:re2j' implementation 'org.slf4j:slf4j-api' + implementation 'net.java.dev.jna:jna' testImplementation project(':cdm-test-utils') testImplementation project(':udunits') testImplementation 'com.google.truth:truth' testImplementation 'org.jsoup:jsoup' + + testRuntimeOnly project(':native-compression:libaec-native') + testRuntimeOnly 'ch.qos.logback:logback-classic' } diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java index ce4e0af311..164c119a37 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java @@ -1,10 +1,16 @@ /* - * Copyright (c) 1998-2021 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package ucar.nc2.grib.grib2; +import static edu.ucar.unidata.compression.jna.libaec.LibAec.AEC_OK; + +import com.sun.jna.Memory; +import edu.ucar.unidata.compression.jna.libaec.LibAec; +import edu.ucar.unidata.compression.jna.libaec.LibAec.AecStream; +import java.nio.ByteBuffer; import javax.annotation.Nullable; import ucar.nc2.grib.GribNumbers; import ucar.nc2.grib.GribUtils; @@ -65,7 +71,7 @@ public class Grib2DataReader { } /* - * Code Table Code table 5.0 - Data representation template number (5.0) + * Code table 5.0 - Data representation template number (5.0) * 0: Grid point data - simple packing * 1: Matrix value at grid point - simple packing * 2: Grid point data - complex packing @@ -73,10 +79,12 @@ public class Grib2DataReader { * 4: Grid point data - IEEE floating point data * 40: Grid point data - JPEG 2000 code stream format * 41: Grid point data - Portable Network Graphics (PNG) + * 42: Grid point data - CCSDS recommended lossless compression * 50: Spectral data - simple packing * 51: Spherical harmonics data - complex packing * 61: Grid point data - simple packing with logarithm pre-processing * 200: Run length packing with level values + * 50002: Second order packing * 65535: Missing */ @@ -111,6 +119,9 @@ public float[] getData(RandomAccessFile raf, Grib2SectionBitMap bitmapSection, G case 41: data = getData41(raf, (Grib2Drs.Type0) gdrs); break; + case 42: + data = getData42(raf, (Grib2Drs.Type42) gdrs); + break; case 50002: data = getData50002(raf, (Grib2Drs.Type50002) gdrs); break; @@ -1073,6 +1084,80 @@ private float[] getData50002(RandomAccessFile raf, Grib2Drs.Type50002 gdrs) thro } + private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOException { + byte[] decodedData; + + // read CCSDS encoded stream from message + int encodedLength = dataLength - 5; + byte[] inputData = new byte[encodedLength]; + raf.readFully(inputData); + + int nbytesPerSample = (gdrs.numberOfBits + 7) / 8; + + try (Memory inputMemory = new Memory(encodedLength); + Memory outputMemory = new Memory((long) nbytesPerSample * totalNPoints)) { + + // set encoding parameters + AecStream aecStreamDecode = AecStream.create(gdrs.numberOfBits, gdrs.blockSize, gdrs.referenceSampleInterval, + gdrs.compressionOptionsMask); + + // load data from grib message into memory + inputMemory.write(0, inputData, 0, inputData.length); + + aecStreamDecode.setInputMemory(inputMemory); + aecStreamDecode.setOutputMemory(outputMemory); + + // decode + int ok = LibAec.aec_buffer_decode(aecStreamDecode); + if (ok != AEC_OK) { + System.out.printf("AEC Error: %s%n", ok); + } + + // read decoded data from native memory + decodedData = new byte[nbytesPerSample * totalNPoints]; + outputMemory.read(0, decodedData, 0, decodedData.length); + } + + // will use this to read out a long value using nbytesPerSample bytes + // see long getNextLong(ByteBuffer bb, int numberOfBytes) + ByteBuffer bb = ByteBuffer.wrap(decodedData); + + // decode following regulation 92.9.4, Note 4 + int D = gdrs.decimalScaleFactor; + float DD = (float) Math.pow((double) 10, (double) D); + float R = gdrs.referenceValue; + int E = gdrs.binaryScaleFactor; + float EE = (float) Math.pow(2.0, (double) E); + float[] data = new float[decodedData.length]; + if (bitmap == null) { + for (int i = 0; i < totalNPoints; i++) { + data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD; + } + } else { + for (int i = 0; i < totalNPoints; i++) { + if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) { + data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD; + } else { + data[i] = staticMissingValue; + } + } + } + return data; + } + + private long getNextLong(ByteBuffer bb, int numberOfBytes) throws IOException { + switch (numberOfBytes) { + case 1: + return Byte.toUnsignedLong(bb.get()); + case 2: + return Short.toUnsignedLong(bb.getShort()); + case 4: + return Integer.toUnsignedLong(bb.getInt()); + default: + throw new IOException("Invalid number of bytes per sample for GDR42: " + numberOfBytes); + } + } + /* * * Flag table 3.4 – Scanning mode diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Drs.java b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Drs.java index 8dda0d266f..96ea4e8e6c 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Drs.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Drs.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2025 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -22,15 +22,17 @@ public abstract class Grib2Drs { public static Grib2Drs factory(int template, RandomAccessFile raf) throws IOException { switch (template) { - case 0: - case 41: + case 0: // simple packing + case 41: // PNG return new Type0(raf); - case 2: + case 2: // complex packing return new Type2(raf); - case 3: + case 3: // complex packing and spatial differencing return new Type3(raf); - case 40: + case 40: // JPEG2000 return new Type40(raf); + case 42: // CCSDS + return new Type42(raf); case 50002: // ECMWF's second order packing return new Type50002(raf); default: @@ -441,6 +443,63 @@ public int hashCode() { } } + + /* + * Data representation template 5.42 – Grid point data - CCSDS recommended lossless compression + * Note: For most templates, details of the packing process are described in Regulation 92.9.4. + * Octet No. Contents + * 12-15 Reference value (R) (IEEE 32-bit floating-point value) + * 16-17 Binary scale factor (E) + * 18-19 Decimal scale factor (D) + * 20 Number of bits required to hold the resulting scaled and referenced data values. (see Note 1) + * 21 Type of original field values (see Code Table 5.1) + * --- additions to Type0 below --- + * 22 CCSDS compression options mask (see Note 3) + * 23 Block size + * 24-25 Reference sample interval + */ + public static class Type42 extends Type0 { + + int compressionOptionsMask, blockSize, referenceSampleInterval; + + Type42(RandomAccessFile raf) throws IOException { + super(raf); + this.compressionOptionsMask = raf.read(); + this.blockSize = raf.read(); + this.referenceSampleInterval = GribNumbers.uint2(raf); + System.out.println("hello"); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("compressionOptionsMask", compressionOptionsMask) + .add("blockSize", blockSize).add("referenceSampleInterval", referenceSampleInterval).toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + + Type42 type2 = (Type42) o; + + if (compressionOptionsMask != type2.compressionOptionsMask) { + return false; + } + if (blockSize != type2.blockSize) { + return false; + } + return referenceSampleInterval == type2.referenceSampleInterval; + } + } + // pull request #52 "lost-carrier" jkaehler@meteomatics.com public static class Type50002 extends Grib2Drs { diff --git a/grib/src/test/data/sref.pds2.drs42.grib2 b/grib/src/test/data/sref.pds2.drs42.grib2 new file mode 100644 index 0000000000..c41e9db58c Binary files /dev/null and b/grib/src/test/data/sref.pds2.drs42.grib2 differ diff --git a/grib/src/test/java/ucar/nc2/grib/grib2/TestDrs42.java b/grib/src/test/java/ucar/nc2/grib/grib2/TestDrs42.java new file mode 100644 index 0000000000..66f174ecb0 --- /dev/null +++ b/grib/src/test/java/ucar/nc2/grib/grib2/TestDrs42.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2025 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package ucar.nc2.grib.grib2; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; +import java.util.Formatter; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import ucar.ma2.Array; +import ucar.ma2.MAMath; +import ucar.ma2.MAMath.MinMax; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.Variable; +import ucar.nc2.util.CompareNetcdf2; +import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.category.NeedsCdmUnitTest; + +public class TestDrs42 { + + @Test + public void CCSDS1byte() throws IOException { + String origFile = TestDir.localTestDataDir + "sref.pds2.grib2"; + // drs42File is origFile convert from Simple Packing to CCSDS by ecCodes + // grib_set -w isGridded=1 -r -s packingType=grid_ccsds sref.pds2.grib2 sref.pds2.drs42.grib2 + // This file uses 8 bits per sample in the CCSDS configuration + String drs42File = TestDir.localTestDataDir + "sref.pds2.drs42.grib2"; + final String variableName = "u-component_of_wind_height_above_ground_weightedMean"; + + // stats from ecCodes + final float expectedMax = 13.4F; + final float expectedMin = -9.9F; + final float expectedAverage = 0.246189F; + + final int expectedLength = 23865; + final float tol = 1e-6F; + + try (NetcdfFile nc42 = NetcdfFiles.open(drs42File)) { + Variable v = nc42.findVariable(variableName); + assertThat(v != null).isTrue(); + Array data = v.read(); + + assertThat(data).isNotNull(); + assertThat(data.getSize()).isEqualTo(expectedLength); + MinMax extremes = MAMath.getMinMax(data); + assertThat(extremes.max).isWithin(tol).of(expectedMax); + assertThat(extremes.min).isWithin(tol).of(expectedMin); + assertThat(MAMath.sumDouble(data) / data.getSize()).isWithin(tol).of(expectedAverage); + + // compare repacked ccsds data with original data + try (NetcdfFile ncOrig = NetcdfFiles.open(origFile)) { + Formatter f = new Formatter(); + CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true); + boolean ok = compare.compare(ncOrig, nc42, null); + System.out.printf("%s %s%n", ok ? "OK" : "NOT OK", f); + assertThat(ok).isTrue(); + } + } + } + + @Test + @Category(NeedsCdmUnitTest.class) + public void checkVariable2Bytes() throws IOException { + // This file uses 16 bits per sample in the CCSDS configuration + String drs42File = TestDir.cdmUnitTestDir + "formats/grib2/drs42/" + + "icon-eu_europe_regular-lat-lon_single-level_2025031912_014_T_2M_CCSDS.grib2"; + final String variableName = "Temperature_height_above_ground"; + + // stats from ecCodes + final float expectedMax = 298.187F; + final float expectedMin = 255.639F; + final float expectedAverage = 278.009F; + + final int expectedLength = 904689; + final float tol = 1e-3F; + + try (NetcdfFile nc42 = NetcdfFiles.open(drs42File)) { + Variable v = nc42.findVariable(variableName); + assertThat(v != null).isTrue(); + Array data = v.read(); + + assertThat(data).isNotNull(); + assertThat(data.getSize()).isEqualTo(expectedLength); + MinMax extremes = MAMath.getMinMax(data); + assertThat(extremes.max).isWithin(tol).of(expectedMax); + assertThat(extremes.min).isWithin(tol).of(expectedMin); + assertThat(MAMath.sumDouble(data) / data.getSize()).isWithin(tol).of(expectedAverage); + } + } +} diff --git a/native-compression/libaec-jna/src/test/java/edu/ucar/unidata/compression/jna/libaec/TestLibAec.java b/native-compression/libaec-jna/src/test/java/edu/ucar/unidata/compression/jna/libaec/TestLibAec.java index 6824c6c7b6..c6f3a894b1 100644 --- a/native-compression/libaec-jna/src/test/java/edu/ucar/unidata/compression/jna/libaec/TestLibAec.java +++ b/native-compression/libaec-jna/src/test/java/edu/ucar/unidata/compression/jna/libaec/TestLibAec.java @@ -81,7 +81,7 @@ public void aecBufferDecode() { try (Memory inputMemory = new Memory(expectedEncoded.length * Byte.BYTES); Memory outputMemory = new Memory(origData.length * Integer.BYTES)) { // set encoding parameters - AecStream aecStreamEncode = AecStream.create(bitsPerSample, blockSize, rsi, flags); + AecStream aecStreamDecode = AecStream.create(bitsPerSample, blockSize, rsi, flags); // load expected encoded data into native memory byte[] expectedEncodedBytes = new byte[expectedEncoded.length]; @@ -90,18 +90,18 @@ public void aecBufferDecode() { } inputMemory.write(0, expectedEncodedBytes, 0, expectedEncodedBytes.length); - aecStreamEncode.setInputMemory(inputMemory); - aecStreamEncode.setOutputMemory(outputMemory); + aecStreamDecode.setInputMemory(inputMemory); + aecStreamDecode.setOutputMemory(outputMemory); // decode - int ok = LibAec.aec_buffer_decode(aecStreamEncode); + int ok = LibAec.aec_buffer_decode(aecStreamDecode); assertThat(ok).isEqualTo(AEC_OK); // check expected number of bytes decoded - assertThat(aecStreamEncode.total_out.intValue()).isEqualTo(origData.length * Integer.BYTES); + assertThat(aecStreamDecode.total_out.intValue()).isEqualTo(origData.length * Integer.BYTES); - // read encoded data from native memory - int[] decodedData = new int[aecStreamEncode.total_out.intValue() / Integer.BYTES]; + // read decoded data from native memory + int[] decodedData = new int[aecStreamDecode.total_out.intValue() / Integer.BYTES]; outputMemory.read(0, decodedData, 0, decodedData.length); // compare decoded data to original values @@ -150,23 +150,23 @@ public void roundTrip() { Memory outputMemory = new Memory(origData.length * Integer.BYTES)) { // set encoding parameters - AecStream aecStreamEncode = AecStream.create(bitsPerSample, blockSize, rsi, flags); + AecStream aecStreamDecode = AecStream.create(bitsPerSample, blockSize, rsi, flags); // load encoded data into memory inputMemory.write(0, encodedData, 0, encodedData.length); - aecStreamEncode.setInputMemory(inputMemory); - aecStreamEncode.setOutputMemory(outputMemory); + aecStreamDecode.setInputMemory(inputMemory); + aecStreamDecode.setOutputMemory(outputMemory); // decode - int ok = LibAec.aec_buffer_decode(aecStreamEncode); + int ok = LibAec.aec_buffer_decode(aecStreamDecode); assertThat(ok).isEqualTo(AEC_OK); // check expected number of bytes decoded - assertThat(aecStreamEncode.total_out.intValue()).isEqualTo(origData.length * Integer.BYTES); + assertThat(aecStreamDecode.total_out.intValue()).isEqualTo(origData.length * Integer.BYTES); - // read encoded data from native memory - int[] decodedData = new int[aecStreamEncode.total_out.intValue() / Integer.BYTES]; + // read decoded data from native memory + int[] decodedData = new int[aecStreamDecode.total_out.intValue() / Integer.BYTES]; outputMemory.read(0, decodedData, 0, decodedData.length); // compare decoded data to original values diff --git a/native-compression/libaec-jna/src/test/java/edu/ucar/unidata/compression/jna/libaec/TestLibAecMultithreaded.java b/native-compression/libaec-jna/src/test/java/edu/ucar/unidata/compression/jna/libaec/TestLibAecMultithreaded.java new file mode 100644 index 0000000000..4dfbf8c4e9 --- /dev/null +++ b/native-compression/libaec-jna/src/test/java/edu/ucar/unidata/compression/jna/libaec/TestLibAecMultithreaded.java @@ -0,0 +1,217 @@ +/* + * Copyright (c) 2025 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package edu.ucar.unidata.compression.jna.libaec; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static edu.ucar.unidata.compression.jna.libaec.LibAec.AEC_DATA_3BYTE; +import static edu.ucar.unidata.compression.jna.libaec.LibAec.AEC_DATA_MSB; +import static edu.ucar.unidata.compression.jna.libaec.LibAec.AEC_DATA_PREPROCESS; +import static edu.ucar.unidata.compression.jna.libaec.LibAec.AEC_DATA_SIGNED; +import static edu.ucar.unidata.compression.jna.libaec.LibAec.AEC_OK; +import static edu.ucar.unidata.compression.jna.libaec.LibAec.AEC_RESTRICTED; + +import com.sun.jna.Memory; +import edu.ucar.unidata.compression.jna.libaec.LibAec.AecStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; +import org.junit.Test; + +/** + * Test multithreaded access to native libaec by round tripping data in multiple threads + */ +public class TestLibAecMultithreaded { + + AtomicReference failed = new AtomicReference<>(); + CountDownLatch startupLatch, readyLatch, finishedLatch; + boolean testDebugMessages = true; + + static class AecTestCase { + int[] origData; + int bitsPerSample, blockSize, rsi, flags; + + AecTestCase(int[] origData, int bitsPerSample, int blockSize, int rsi, int flags) { + this.origData = origData; + this.bitsPerSample = bitsPerSample; + this.blockSize = blockSize; + this.rsi = rsi; + this.flags = flags; + } + } + + public static int[] createRandom(int n) { + Random r = new Random(); + int min = -10000; + int max = 10000; + return IntStream.generate(() -> r.nextInt(max - min) + min).limit(n).toArray(); + } + + public class RoundTripRunnable implements Runnable { + + AecTestCase myTestCase; + + RoundTripRunnable(AecTestCase testCase) { + this.myTestCase = testCase; + } + + @Override + public void run() { + if (testDebugMessages) { + System.out.println(Thread.currentThread().getId() + ", awaiting execution signal"); + } + try { + readyLatch.countDown(); + boolean startupReady = startupLatch.await(1, TimeUnit.SECONDS); + assertWithMessage("test startup took too long").that(startupReady).isTrue(); + } catch (InterruptedException e) { + failed.set(new AssertionError("test startup took too long", e));; + } + if (testDebugMessages) { + System.out.println(Thread.currentThread().getId() + ", executing run() method!"); + } + // byte array to hold the encoded data + byte[] encodedData; + + // convert input data into byte array + ByteBuffer bb = ByteBuffer.allocate(myTestCase.origData.length * Integer.BYTES); + bb.order(ByteOrder.nativeOrder()); + bb.asIntBuffer().put(myTestCase.origData); + byte[] inputData = bb.array(); + final int maxTotalOutBytes = myTestCase.origData.length * Integer.BYTES * 67 / 64 + 256; + try { + // initialize native memory to hold input and output data + try (Memory inputMemory = new Memory(inputData.length * Byte.BYTES); + Memory outputMemory = new Memory(maxTotalOutBytes)) { + + // set encoding parameters + AecStream aecStreamDecode = + AecStream.create(myTestCase.bitsPerSample, myTestCase.blockSize, myTestCase.rsi, myTestCase.flags); + + // load input data into native memory + inputMemory.write(0, inputData, 0, inputData.length); + aecStreamDecode.setInputMemory(inputMemory); + + aecStreamDecode.setOutputMemory(outputMemory); + + // encode + int ok = LibAec.aec_buffer_encode(aecStreamDecode); + assertThat(ok).isEqualTo(AEC_OK); + + // read encoded data from native memory + encodedData = new byte[aecStreamDecode.total_out.intValue()]; + outputMemory.read(0, encodedData, 0, encodedData.length); + } + + assertThat(encodedData).isNotNull(); + + try (Memory inputMemory = new Memory(encodedData.length * Byte.BYTES); + Memory outputMemory = new Memory((long) myTestCase.origData.length * Integer.BYTES)) { + + // set encoding parameters + AecStream aecStreamDecode = + AecStream.create(myTestCase.bitsPerSample, myTestCase.blockSize, myTestCase.rsi, myTestCase.flags); + + // load encoded data into memory + inputMemory.write(0, encodedData, 0, encodedData.length); + + aecStreamDecode.setInputMemory(inputMemory); + aecStreamDecode.setOutputMemory(outputMemory); + + // decode + int ok = LibAec.aec_buffer_decode(aecStreamDecode); + assertThat(ok).isEqualTo(AEC_OK); + + // check expected number of bytes decoded + assertThat(aecStreamDecode.total_out.intValue()).isEqualTo(myTestCase.origData.length * Integer.BYTES); + + // read decoded data from native memory + int[] decodedData = new int[aecStreamDecode.total_out.intValue() / Integer.BYTES]; + outputMemory.read(0, decodedData, 0, decodedData.length); + + // compare decoded data to original values + for (int i = 0; i < decodedData.length; i++) { + assertThat(decodedData[i]).isEqualTo(myTestCase.origData[i]); + } + } + if (testDebugMessages) { + System.out.println(Thread.currentThread().getId() + ", finished!"); + } + } catch (AssertionError ae) { + failed.set(ae); + } finally { + finishedLatch.countDown(); + } + } + } + + void runTest(AecTestCase[] myTestCases) { + startupLatch = new CountDownLatch(1); + if (testDebugMessages) { + System.out.println("Main thread is: " + Thread.currentThread().getName()); + } + + + finishedLatch = new CountDownLatch(myTestCases.length); + readyLatch = new CountDownLatch(myTestCases.length); + for (AecTestCase myTestCase : myTestCases) { + Thread t = new Thread(new RoundTripRunnable(myTestCase)); + t.start(); + } + // ensure threaded tests start running at the same time + try { + boolean testThreadsReady = readyLatch.await(1, TimeUnit.SECONDS); + assertWithMessage("test threads took too long to prepare").that(testThreadsReady).isTrue(); + } catch (InterruptedException e) { + failed.set(new AssertionError("test threads took too long to prepare", e));; + } + // trigger threaded tests to start running + if (testDebugMessages) { + System.out.println("Start testing threads from: " + Thread.currentThread().getName()); + } + startupLatch.countDown(); + // wait for tests to finish + try { + boolean threadsComplete = finishedLatch.await(5, TimeUnit.SECONDS); + assertWithMessage("test threads failed to complete").that(threadsComplete).isTrue(); + } catch (InterruptedException e) { + failed.set(new AssertionError("test threads took too long to complete", e));; + } + if (failed.get() != null) { + throw failed.get(); + } + } + + @Test + public void testMultithreadedDifferentParams() { + AecTestCase[] differentParamTestCases = + new AecTestCase[] {new AecTestCase(createRandom(2500), 32, 32, 64, AEC_DATA_SIGNED), + new AecTestCase(createRandom(210), 32, 16, 128, AEC_DATA_MSB | AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(24), 32, 8, 128, AEC_DATA_3BYTE | AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(2400), 32, 16, 128, AEC_RESTRICTED), + new AecTestCase(createRandom(29), 32, 32, 128, AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(255), 32, 8, 128, AEC_DATA_SIGNED | AEC_DATA_MSB | AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(2045), 32, 16, 128, AEC_DATA_3BYTE)}; + runTest(differentParamTestCases); + } + + @Test + public void testMultiThreadedSameParams() { + AecTestCase[] sameParamTestCases = + new AecTestCase[] {new AecTestCase(createRandom(2500), 32, 16, 128, AEC_DATA_SIGNED | AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(2500), 32, 16, 128, AEC_DATA_SIGNED | AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(2500), 32, 16, 128, AEC_DATA_SIGNED | AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(2500), 32, 16, 128, AEC_DATA_SIGNED | AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(2500), 32, 16, 128, AEC_DATA_SIGNED | AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(2500), 32, 16, 128, AEC_DATA_SIGNED | AEC_DATA_PREPROCESS), + new AecTestCase(createRandom(2500), 32, 16, 128, AEC_DATA_SIGNED | AEC_DATA_PREPROCESS)}; + runTest(sameParamTestCases); + } +} diff --git a/netcdf-java-bom/build.gradle b/netcdf-java-bom/build.gradle index 872e2f063d..bc5524b8f0 100644 --- a/netcdf-java-bom/build.gradle +++ b/netcdf-java-bom/build.gradle @@ -23,6 +23,8 @@ dependencies { api project(':grib') api project(':httpservices') api project(':legacy') + api project(':native-compression:libaec-jna') + api project(':native-compression:libaec-native') api project(':netcdf4') api project(':opendap') api project(':dap4')