From 0780cbae9d580b4a3f31ea0acc908791ea4ef836 Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Thu, 5 Dec 2019 10:40:15 +0800 Subject: [PATCH 01/13] [POAE7-11](add lucene-qat-wrapper,ZhangJie, Reviewed by XuCheng) --- lucene_qat_wrapper/build/native/Makefile | 63 ++++ lucene_qat_wrapper/pom.xml | 236 ++++++++++++ .../intel/qat/conf/QatConfigurationKeys.java | 54 +++ .../com/intel/qat/jni/QatCompressorJNI.java | 357 ++++++++++++++++++ .../com/intel/qat/jni/QatDecompressorJNI.java | 331 ++++++++++++++++ .../intel/qat/util/QatNativeCodeLoader.java | 78 ++++ .../src/main/native/QatCompressorJNI.c | 185 +++++++++ .../src/main/native/QatDecompressorJNI.c | 158 ++++++++ .../src/main/native/com_intel_qat.h | 124 ++++++ lucene_qat_wrapper/src/main/native/config.h | 21 ++ lucene_qat_wrapper/src/main/native/qatcodec.h | 43 +++ lucene_qat_wrapper/src/main/native/util.c | 71 ++++ pom.xml | 1 + 13 files changed, 1722 insertions(+) create mode 100644 lucene_qat_wrapper/build/native/Makefile create mode 100644 lucene_qat_wrapper/pom.xml create mode 100644 lucene_qat_wrapper/src/main/java/com/intel/qat/conf/QatConfigurationKeys.java create mode 100644 lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java create mode 100644 lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java create mode 100644 lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java create mode 100644 lucene_qat_wrapper/src/main/native/QatCompressorJNI.c create mode 100644 lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c create mode 100644 lucene_qat_wrapper/src/main/native/com_intel_qat.h create mode 100644 lucene_qat_wrapper/src/main/native/config.h create mode 100644 lucene_qat_wrapper/src/main/native/qatcodec.h create mode 100644 lucene_qat_wrapper/src/main/native/util.c diff --git a/lucene_qat_wrapper/build/native/Makefile b/lucene_qat_wrapper/build/native/Makefile new file mode 100644 index 0000000..d0add31 --- /dev/null +++ b/lucene_qat_wrapper/build/native/Makefile @@ -0,0 +1,63 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# A function designed to check whether every environment variable in a list is +# defined. +check = $(foreach var,$(1),$(if $($(var)),,$(error Please set $(var)))) + +# These were the compiler flags used when the native part was built into the +# Hadoop native library. It has been decoupled since, so one or more of these +# flags might be redundant. +FLAGS = -g -Wall -O2 -D_REENTRANT -D_GNU_SOURCE -D_LARGEFILE_SOURCE \ + -D_FILE_OFFSET_BITS=64 \ + -fstack-protector-all \ + -D_FORTIFY_SOURCE=2 \ + -Wformat -Wformat-security -Werror=format-security + +INC = -I $(JAVA_HOME)/include \ + -I $(JAVA_HOME)/include/linux \ + -I $(QATZIPSRC)/include \ + -I javah \ + -I ../../src/main/native \ + -I . + +libqatcodec.so: QatCompressorJNI.o QatDecompressorJNI.o util.o + gcc -fpic -shared -o ../../target/libqatcodec.so -ldl QatCompressorJNI.o \ + QatDecompressorJNI.o util.o + +QatCompressorJNI.o: ../../src/main/native/QatCompressorJNI.c javah + $(call check,JAVA_HOME ) + gcc -fpic $(FLAGS) $(INC) -c ../../src/main/native/QatCompressorJNI.c + +QatDecompressorJNI.o: ../../src/main/native/QatDecompressorJNI.c javah + $(call check,JAVA_HOME ) + gcc -fpic $(FLAGS) $(INC) -c ../../src/main/native/QatDecompressorJNI.c + +util.o: ../../src/main/native/util.c + $(call check,JAVA_HOME ) + gcc -fpic $(FLAGS) $(INC) -c ../../src/main/native/util.c + +javah: + $(call check,JAVA_HOME) + $(JAVA_HOME)/bin/javah -d javah -cp ../../src/main/java \ + com.intel.qat.jni.QatCompressorJNI \ + com.intel.qat.jni.QatDecompressorJNI + +clean: + rm -fr ../../target/libqatcodec.so QatCompressorJNI.o QatDecompressorJNI.o util.o \ + javah diff --git a/lucene_qat_wrapper/pom.xml b/lucene_qat_wrapper/pom.xml new file mode 100644 index 0000000..7796fb9 --- /dev/null +++ b/lucene_qat_wrapper/pom.xml @@ -0,0 +1,236 @@ + + + + qat-parent + com.intel.qat + 1.0.0 + ../pom.xml + + 4.0.0 + + lucene_qat_wrapper + Apache Lucene QAT Codec + Apache Lucene QAT Codec + jar + + + + UTF-8 + UTF-8 + false + ./ + + 1.8 + + 3.3.9 + 3.6.1 + 1.4.1 + 2.19.1 + 3.0.2 + 1.8 + 3.0.1 + 3.0.0 + 2.10.4 + + 3.0.0 + 2.5.2 + 2.8.2 + 3.0.0 + 2.17 + 4.12 + + + ${project.build.directory}/jars + + + prepare-package + package + + 512m + + + + + mavencentral + maven central repository + http://repo1.maven.org/maven2 + default + + true + warn + + + false + + + + datanucleus + datanucleus maven repository + http://www.datanucleus.org/downloads/maven2 + default + + true + warn + + + false + + + + glassfish-repository + http://maven.glassfish.org/content/groups/glassfish + + false + + + false + + + + sonatype-snapshot + https://oss.sonatype.org/content/repositories/snapshots + + false + + + false + + + + + + + central + https://repo1.maven.org/maven2 + + true + + + false + + + + + + + + org.apache.logging.log4j + log4j-api + 2.11.1 + + + org.apache.logging.log4j + log4j-core + 2.11.1 + + + + junit + junit + ${junit.version} + test + + + + + org.slf4j + slf4j-api + 1.7.2 + + + + org.slf4j + slf4j-log4j12 + 1.7.2 + test + + + + + + + + maven-surefire-plugin + 2.19.1 + + + java.library.path + target + + + ${qatzip.libs} + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.0.2 + + + + test-jar + + + + + + + + + + native + + + !skipNative + + + Linux + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + build-native + compile + + run + + + + Java Home: ${java.home} + + + + + + + + + + clean-native + clean + + run + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/conf/QatConfigurationKeys.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/conf/QatConfigurationKeys.java new file mode 100644 index 0000000..66e3a2f --- /dev/null +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/conf/QatConfigurationKeys.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.qat.conf; + +public class QatConfigurationKeys{ + /** Internal buffer size for qat compressor/decompressors */ + public static final String IO_COMPRESSION_CODEC_QAT_BUFFERSIZE_KEY = + "io.compression.codec.qat.buffersize"; + + /** Default value for IO_COMPRESSION_CODEC_QAT_BUFFERSIZE_KEY */ + public static final int IO_COMPRESSION_CODEC_QAT_BUFFERSIZE_DEFAULT = + 256 * 1024; + + /** + * Whether to use native allocate BB for creating ByteBuffer. + */ + public static final String IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_KEY = + "io.compression.codec.qat.use-native-allocate-bb"; + + public static final boolean IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_DEFAULT = false; + + /** + * Whether to use force pinned for native allocate BB, it is applicable only + * when IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_KEY is enabled. + */ + public static final String IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_FORCE_PINNED_KEY = + "io.compression.codec.qat.native-allocate-bb.force-pinned"; + + public static final boolean IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_FORCE_PINNED_DEFAULT = true; + + /** + * Whether to use numa for native allocate BB, it is applicable only when + * IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_KEY is enabled. + */ + public static final String IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_NUMA_KEY = + "io.compression.codec.qat.native-allocate-bb.numa"; + + public static final boolean IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_NUMA_DEFAULT = false; +} diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java new file mode 100644 index 0000000..f877f2d --- /dev/null +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java @@ -0,0 +1,357 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.intel.qat.jni; + +import java.io.IOException; +import java.nio.Buffer; +import java.nio.ByteBuffer; +//import org.apache.logging.log4j.LogManager; +//import org.apache.logging.log4j.Logger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.intel.qat.conf.QatConfigurationKeys; +import com.intel.qat.util.QatNativeCodeLoader; + + +/** + * @author root + * @date 12/3/19 12:39 PM + * @decsription: A {@link Compressor} based on the qat compression algorithm. + * @link qat-parent + */ + +public class QatCompressorJNI { + + private static final Logger LOG = LoggerFactory.getLogger(QatCompressorJNI.class.getName()); + //private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); + private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; + + // HACK - Use this as a global lock in the JNI layer + @SuppressWarnings({"rawtypes"}) + private static Class clazz = QatCompressorJNI.class; + + private int directBufferSize; + private Buffer compressedDirectBuf = null; + private int uncompressedDirectBufLen; + private Buffer uncompressedDirectBuf = null; + private byte[] userBuf = null; + private int userBufOff = 0, userBufLen = 0; + private boolean finish, finished; + + private long bytesRead = 0L; + private long bytesWritten = 0L; + + private static boolean nativeQatLoaded = false; + + static { + if (QatNativeCodeLoader.isNativeCodeLoaded() && + QatNativeCodeLoader.buildSupportsQat()) { + try { + String value = System.getProperty("QAT_COMPRESS_LEVEL"); + int level = 1; + if (value != null) { + try { + level = Integer.parseInt(value); + if (level < 1 || level > 9) { + level = 1; + LOG.warn("Invalid value for compression level:" + value + + ", value should be in range 1-9." + + " Proceeding with default value as 1."); + } + } catch (NumberFormatException e) { + level = 1; + LOG.warn("Could not parse the value:" + value + + ", compression level should be in range 1-9." + + " Proceeding with default value as 1."); + } + } + initIDs(level); + nativeQatLoaded = true; + } catch (Throwable t) { + LOG.error("failed to load QatCompressor AMAC QatCompressor", t); + } + } + } + + public static boolean isNativeCodeLoaded() { + return nativeQatLoaded; + } + + /** + * Creates a new compressor. + * + * @param directBufferSize size of the direct buffer to be used. + * @param numa + * @param forcePinned + */ + public QatCompressorJNI(int directBufferSize, boolean useNativeAllocateBB, + boolean forcePinned, boolean numa) { + this.directBufferSize = directBufferSize; + if (useNativeAllocateBB) { + LOG.info("Creating ByteBuffer's using nativeAllocateBB."); + try { + uncompressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, + numa, forcePinned); + } catch (Throwable t) { + LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + + " for uncompressed direct ByteBuffer. Creating the uncompressed" + + " ByteBuffer using ByteBuffer.allocateDirect().", t); + uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + } + try { + compressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, + numa, forcePinned); + } catch (Throwable t) { + LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + + " for compressed direct ByteBuffer. Creating the compressed" + + " ByteBuffer using ByteBuffer.allocateDirect().", t); + compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + } + } else { + uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + } + compressedDirectBuf.position(directBufferSize); + } + + /** + * Creates a new compressor with the directBufferSize. + * + * @param directBufferSize + */ + public QatCompressorJNI(int directBufferSize) { + this(directBufferSize, + QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_DEFAULT, + QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_FORCE_PINNED_DEFAULT, + QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_NUMA_DEFAULT); + } + + /** + * Creates a new compressor with the default buffer size. + */ + public QatCompressorJNI() { + this(DEFAULT_DIRECT_BUFFER_SIZE); + } + + /** + * Sets input data for compression. + * This should be called whenever #needsInput() returns + * true indicating that more input data is required. + * + * @param b Input data + * @param off Start offset + * @param len Length + */ + + public synchronized void setInput(byte[] b, int off, int len) { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + finished = false; + + if (len > uncompressedDirectBuf.remaining()) { + // save data; now !needsInput + this.userBuf = b; + this.userBufOff = off; + this.userBufLen = len; + } else { + ((ByteBuffer) uncompressedDirectBuf).put(b, off, len); + uncompressedDirectBufLen = uncompressedDirectBuf.position(); + } + + bytesRead += len; + } + + /** + * If a write would exceed the capacity of the direct buffers, it is set + * aside to be loaded by this function while the compressed data are + * consumed. + */ + synchronized void setInputFromSavedData() { + if (0 >= userBufLen) { + return; + } + finished = false; + + uncompressedDirectBufLen = Math.min(userBufLen, directBufferSize); + ((ByteBuffer) uncompressedDirectBuf).put(userBuf, userBufOff, + uncompressedDirectBufLen); + + // Note how much data is being fed to qat + userBufOff += uncompressedDirectBufLen; + userBufLen -= uncompressedDirectBufLen; + } + + /** + * Does nothing. + */ + + public synchronized void setDictionary(byte[] b, int off, int len) { + // do nothing + } + + /** + * Returns true if the input data buffer is empty and + * #setInput() should be called to provide more input. + * + * @return true if the input data buffer is empty and + * #setInput() should be called in order to provide more input. + */ + + public synchronized boolean needsInput() { + return !(compressedDirectBuf.remaining() > 0 + || uncompressedDirectBuf.remaining() == 0 || userBufLen > 0); + } + + /** + * When called, indicates that compression should end + * with the current contents of the input buffer. + */ + + public synchronized void finish() { + finish = true; + } + + /** + * Returns true if the end of the compressed + * data output stream has been reached. + * + * @return true if the end of the compressed + * data output stream has been reached. + */ + + public synchronized boolean finished() { + // Check if all uncompressed data has been consumed + return (finish && finished && compressedDirectBuf.remaining() == 0); + } + + /** + * Fills specified buffer with compressed data. Returns actual number + * of bytes of compressed data. A return value of 0 indicates that + * needsInput() should be called in order to determine if more input + * data is required. + * + * @param b Buffer for the compressed data + * @param off Start offset of the data + * @param len Size of the buffer + * @return The actual number of bytes of compressed data. + */ + + public synchronized int compress(byte[] b, int off, int len) + throws IOException { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + + // Check if there is compressed data + int n = compressedDirectBuf.remaining(); + if (n > 0) { + n = Math.min(n, len); + ((ByteBuffer) compressedDirectBuf).get(b, off, n); + bytesWritten += n; + return n; + } + + // Re-initialize the qat's output direct-buffer + compressedDirectBuf.clear(); + compressedDirectBuf.limit(0); + if (0 == uncompressedDirectBuf.position()) { + // No compressed data, so we should have !needsInput or !finished + setInputFromSavedData(); + if (0 == uncompressedDirectBuf.position()) { + // Called without data; write nothing + finished = true; + return 0; + } + } + + // Compress data + n = compressBytesDirect(); + compressedDirectBuf.limit(n); + uncompressedDirectBuf.clear(); // qat consumes all buffer input + + // Set 'finished' if qat has consumed all user-data + if (0 == userBufLen) { + finished = true; + } + + // Get atmost 'len' bytes + n = Math.min(n, len); + bytesWritten += n; + ((ByteBuffer) compressedDirectBuf).get(b, off, n); + + return n; + } + + /** + * Resets compressor so that a new set of input data can be processed. + */ + + public synchronized void reset() { + finish = false; + finished = false; + uncompressedDirectBuf.clear(); + uncompressedDirectBufLen = 0; + compressedDirectBuf.clear(); + compressedDirectBuf.limit(0); + userBufOff = userBufLen = 0; + bytesRead = bytesWritten = 0L; + } + + /** + * Return number of bytes given to this compressor since last reset. + */ + + public synchronized long getBytesRead() { + return bytesRead; + } + + /** + * Return number of bytes consumed by callers of compress since last reset. + */ + + public synchronized long getBytesWritten() { + return bytesWritten; + } + + /** + * Closes the compressor and discards any unprocessed input. + */ + + public synchronized void end() { + } + + private native static void initIDs(int level); + + private native int compressBytesDirect(); + + public native static String getLibraryName(); + + public native Object nativeAllocateBB(long capacity, boolean numa, + boolean forcePinned); +} \ No newline at end of file diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java new file mode 100644 index 0000000..e7fcf7c --- /dev/null +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -0,0 +1,331 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intel.qat.jni; + +import java.io.IOException; +import java.nio.Buffer; +import java.nio.ByteBuffer; + +import com.intel.qat.conf.QatConfigurationKeys; +import com.intel.qat.util.QatNativeCodeLoader; + +//import org.apache.logging.log4j.LogManager; +//import org.apache.logging.log4j.Logger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author root + * @date 12/3/19 12:47 PM + * @decsription: A {@link Decompressor} based on the qat compression algorithm. It's not thread-safe. + * @link qat-parent + */ +public class QatDecompressorJNI { + private static final Logger LOG = LoggerFactory.getLogger(QatDecompressorJNI.class.getName()); + //private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); + + private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; + + // HACK - Use this as a global lock in the JNI layer + @SuppressWarnings({"rawtypes"}) + private static Class clazz = QatDecompressorJNI.class; + + private int directBufferSize; + private Buffer compressedDirectBuf = null; + private int compressedDirectBufLen; + private Buffer uncompressedDirectBuf = null; + private byte[] userBuf = null; + private int userBufOff = 0, userBufLen = 0; + private boolean finished; + + private static boolean nativeQatLoaded = false; + + static { + if (QatNativeCodeLoader.isNativeCodeLoaded() && + QatNativeCodeLoader.buildSupportsQat()) { + try { + initIDs(); + nativeQatLoaded = true; + } catch (Throwable t) { + LOG.error("failed to load QatDecompressor", t); + } + } + } + + public static boolean isNativeCodeLoaded() { + return nativeQatLoaded; + } + + /** + * Creates a new decompressor. + * + * @param directBufferSize + * size of the direct buffer to be used. + * @param useNativeAllocateBB + * @param forcePinned + * @param numa + */ + public QatDecompressorJNI(int directBufferSize, boolean useNativeAllocateBB, + boolean forcePinned, boolean numa) { + this.directBufferSize = directBufferSize; + if (useNativeAllocateBB) { + LOG.info("Creating ByteBuffer's using nativeAllocateBB."); + try { + uncompressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, + numa, forcePinned); + } catch (Throwable t) { + LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + + " for uncompressed direct ByteBuffer. Creating the uncompressed" + + " ByteBuffer using ByteBuffer.allocateDirect().", t); + uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + } + try { + compressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, + numa, forcePinned); + } catch (Throwable t) { + LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + + " for compressed direct ByteBuffer. Creating the compressed" + + " ByteBuffer using ByteBuffer.allocateDirect().", t); + compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + } + } else { + uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + } + uncompressedDirectBuf.position(directBufferSize); + } + + /** + * Creates a new decompressor with the directBufferSize. + * + * @param directBufferSize + */ + public QatDecompressorJNI(int directBufferSize) { + this(directBufferSize, + QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_USE_NATIVE_ALLOCATE_BB_DEFAULT, + QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_FORCE_PINNED_DEFAULT, + QatConfigurationKeys.IO_COMPRESSION_CODEC_QAT_NATIVE_ALLOCATE_BB_NUMA_DEFAULT); + } + + /** + * Creates a new decompressor with the default buffer size. + */ + public QatDecompressorJNI() { + this(DEFAULT_DIRECT_BUFFER_SIZE); + } + + /** + * Sets input data for decompression. + * This should be called if and only if {@link #needsInput()} returns + * true indicating that more input data is required. + * (Both native and non-native versions of various Decompressors require + * that the data passed in via b[] remain unmodified until + * the caller is explicitly notified--via {@link #needsInput()}--that the + * buffer may be safely modified. With this requirement, an extra + * buffer-copy can be avoided.) + * + * @param b Input data + * @param off Start offset + * @param len Length + */ + + public void setInput(byte[] b, int off, int len) { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + + this.userBuf = b; + this.userBufOff = off; + this.userBufLen = len; + + setInputFromSavedData(); + + // Reinitialize qat's output direct-buffer + uncompressedDirectBuf.limit(directBufferSize); + uncompressedDirectBuf.position(directBufferSize); + } + + /** + * If a write would exceed the capacity of the direct buffers, it is set + * aside to be loaded by this function while the compressed data are + * consumed. + */ + void setInputFromSavedData() { + compressedDirectBufLen = Math.min(userBufLen, directBufferSize); + + // Reinitialize qat's input direct buffer + compressedDirectBuf.rewind(); + ((ByteBuffer) compressedDirectBuf).put(userBuf, userBufOff, + compressedDirectBufLen); + + // Note how much data is being fed to qat + userBufOff += compressedDirectBufLen; + userBufLen -= compressedDirectBufLen; + } + + /** + * Does nothing. + */ + + public void setDictionary(byte[] b, int off, int len) { + // do nothing + } + + /** + * Returns true if the input data buffer is empty and + * {@link #setInput(byte[], int, int)} should be called to + * provide more input. + * + * @return true if the input data buffer is empty and + * {@link #setInput(byte[], int, int)} should be called in + * order to provide more input. + */ + + public boolean needsInput() { + // Consume remaining compressed data? + if (uncompressedDirectBuf.remaining() > 0) { + return false; + } + + // Check if qat has consumed all input + if (compressedDirectBufLen <= 0) { + // Check if we have consumed all user-input + if (userBufLen <= 0) { + return true; + } else { + setInputFromSavedData(); + } + } + + return false; + } + + /** + * Returns false. + * + * @return false. + */ + + public boolean needsDictionary() { + return false; + } + + /** + * Returns true if the end of the decompressed + * data output stream has been reached. + * + * @return true if the end of the decompressed + * data output stream has been reached. + */ + + public boolean finished() { + return (finished && uncompressedDirectBuf.remaining() == 0); + } + + /** + * Fills specified buffer with uncompressed data. Returns actual number + * of bytes of uncompressed data. A return value of 0 indicates that + * {@link #needsInput()} should be called in order to determine if more + * input data is required. + * + * @param b Buffer for the compressed data + * @param off Start offset of the data + * @param len Size of the buffer + * @return The actual number of bytes of compressed data. + * @throws IOException + */ + + public int decompress(byte[] b, int off, int len) + throws IOException { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + + int n = 0; + + // Check if there is uncompressed data + n = uncompressedDirectBuf.remaining(); + if (n > 0) { + n = Math.min(n, len); + ((ByteBuffer) uncompressedDirectBuf).get(b, off, n); + return n; + } + if (compressedDirectBufLen > 0) { + // Re-initialize the qat's output direct buffer + uncompressedDirectBuf.rewind(); + uncompressedDirectBuf.limit(directBufferSize); + + // Decompress data + n = decompressBytesDirect(); + uncompressedDirectBuf.limit(n); + + if (userBufLen <= 0) { + finished = true; + } + + // Get atmost 'len' bytes + n = Math.min(n, len); + ((ByteBuffer) uncompressedDirectBuf).get(b, off, n); + } + + return n; + } + + /** + * Returns 0. + * + * @return 0. + */ + + public int getRemaining() { + // Never use this function in BlockDecompressorStream. + return 0; + } + + + public void reset() { + finished = false; + compressedDirectBufLen = 0; + uncompressedDirectBuf.limit(directBufferSize); + uncompressedDirectBuf.position(directBufferSize); + userBufOff = userBufLen = 0; + } + + /** + * Resets decompressor and input and output buffers so that a new set of + * input data can be processed. + */ + public void end() { + // do nothing + } + + private native static void initIDs(); + + private native int decompressBytesDirect(); + + public native Object nativeAllocateBB(long capacity, boolean numa, + boolean forcePinned); + +} diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java new file mode 100644 index 0000000..502b3cb --- /dev/null +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.intel.qat.util; + +//import org.apache.logging.log4j.LogManager; +//import org.apache.logging.log4j.Logger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/** + * A helper to load the native qat code i.e. libqat.so. + * This handles the fallback to either the bundled libhadoop-Linux-i386-32.so + * or the default java implementations where appropriate. + * + */ + +public class QatNativeCodeLoader { + private static final Logger LOG = LoggerFactory.getLogger(QatNativeCodeLoader.class); + //private static final Logger LOG = LogManager.getLogger(QatNativeCodeLoader.class); + private static boolean nativeCodeLoaded = false; + + static { + // Try to load native qat library and set fallback flag appropriately + if(LOG.isDebugEnabled()) { + LOG.debug("Trying to load the custom-built native-qat library..."); + } + try { + System.loadLibrary("qatcodec"); + LOG.debug("Loaded the native-qat library"); + nativeCodeLoaded = true; + } catch (Throwable t) { + // Ignore failure to load + if(LOG.isDebugEnabled()) { + LOG.debug("Failed to load native-qat with error: " + t); + LOG.debug("java.library.path=" + + System.getProperty("java.library.path")); + } + } + + if (!nativeCodeLoaded) { + LOG.warn("Unable to load native-qat library for your platform... " + + "using builtin-java classes where applicable"); + } + } + + /** + * Check if native-qat code is loaded for this platform. + * + * @return true if native-qat is loaded, + * else false + */ + public static boolean isNativeCodeLoaded() { + return nativeCodeLoaded; + } + + /** + * Returns true only if this build was compiled with support for qat. + */ + public static native boolean buildSupportsQat(); + + public static native String getLibraryName(); +} diff --git a/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c new file mode 100644 index 0000000..f9aa149 --- /dev/null +++ b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include "qatcodec.h" + +#if defined LUCENE_QAT_LIBRARY + +#include +#include +#include +#include + +#ifdef WINDOWS +#include "winutils.h" +#endif + +#include "com_intel_qat_jni_QatCompressorJNI.h" + +#define JINT_MAX 0x7fffffff +#define qaePinnedMemAlloc(x, y) qaeMemAllocNUMA((x), (y), 8) +#define qaePinnedMemFree(x) qaeMemFreeNUMA((void **)&(x)) + +static jfieldID QatCompressorJNI_clazz; +static jfieldID QatCompressorJNI_uncompressedDirectBuf; +static jfieldID QatCompressorJNI_uncompressedDirectBufLen; +static jfieldID QatCompressorJNI_compressedDirectBuf; +static jfieldID QatCompressorJNI_directBufferSize; + +__thread QzSession_T g_qzCompressSession = { + .internal = NULL, +}; + +#ifdef UNIX +unsigned char* (*dlsym_qzMalloc)(int, int, int); +static int (*dlsym_qzCompress)(QzSession_T *sess, const unsigned char* src, + unsigned int* src_len, unsigned char* dest, unsigned int* dest_len, + unsigned int last); +int (*dlsym_qzGetDefaults)(QzSessionParams_T *defaults); +int (*dlsym_qzSetDefaults)(QzSessionParams_T *defaults); +#endif + +JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatCompressorJNI_initIDs +(JNIEnv *env, jclass clazz, jint level){ + QzSessionParams_T params; +#ifdef UNIX + // Load libqatzip.so + void *libqatzip = dlopen("libqatzip.so", RTLD_LAZY | RTLD_GLOBAL); + if (!libqatzip) { + char msg[1000]; + snprintf(msg, 1000, "%s (%s)!", "Cannot load " LUCENE_QAT_LIBRARY, dlerror()); + THROW(env, "java/lang/UnsatisfiedLinkError1", msg); + return; + } +#endif + + // Locate the requisite symbols from libqatzip.so +#ifdef UNIX + dlerror(); // Clear any existing error + LOAD_DYNAMIC_SYMBOL(dlsym_qzCompress, env, libqatzip, "qzCompress"); + LOAD_DYNAMIC_SYMBOL(dlsym_qzMalloc, env, libqatzip, "qzMalloc"); + LOAD_DYNAMIC_SYMBOL(dlsym_qzGetDefaults, env, libqatzip, "qzGetDefaults"); + LOAD_DYNAMIC_SYMBOL(dlsym_qzSetDefaults, env, libqatzip, "qzSetDefaults"); +#endif + + QatCompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz", + "Ljava/lang/Class;"); + QatCompressorJNI_uncompressedDirectBuf = (*env)->GetFieldID(env, clazz, + "uncompressedDirectBuf", + "Ljava/nio/Buffer;"); + QatCompressorJNI_uncompressedDirectBufLen = (*env)->GetFieldID(env, clazz, + "uncompressedDirectBufLen", "I"); + QatCompressorJNI_compressedDirectBuf = (*env)->GetFieldID(env, clazz, + "compressedDirectBuf", + "Ljava/nio/Buffer;"); + QatCompressorJNI_directBufferSize = (*env)->GetFieldID(env, clazz, + "directBufferSize", "I"); + dlsym_qzGetDefaults(¶ms); + params.comp_lvl = level; + fprintf(stderr, "compression level is %d, tid is %d\n", level, syscall(__NR_gettid)); + dlsym_qzSetDefaults(¶ms); +} + +JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatCompressorJNI_compressBytesDirect +(JNIEnv *env, jobject thisj){ + const unsigned char* uncompressed_bytes; + unsigned char* compressed_bytes; + int ret; + // Get members of QatCompressorJNI + jobject clazz = (*env)->GetStaticObjectField(env, thisj, QatCompressorJNI_clazz); + jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, QatCompressorJNI_uncompressedDirectBuf); + jint uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, QatCompressorJNI_uncompressedDirectBufLen); + jobject compressed_direct_buf = (*env)->GetObjectField(env, thisj, QatCompressorJNI_compressedDirectBuf); + jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, QatCompressorJNI_directBufferSize); + unsigned int buf_len; + unsigned int src_len; + + // Get the input direct buffer + LOCK_CLASS(env, clazz, "QatCompressorJNI"); + uncompressed_bytes = (const unsigned char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); + UNLOCK_CLASS(env, clazz, "QatCompressorJNI"); + + if (uncompressed_bytes == 0) { + return (jint)0; + } + + // Get the output direct buffer + LOCK_CLASS(env, clazz, "QatCompressorJNI"); + compressed_bytes = (unsigned char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); + UNLOCK_CLASS(env, clazz, "QatCompressorJNI"); + + if (compressed_bytes == 0) { + return (jint)0; + } + + /* size_t should always be 4 bytes or larger. */ + buf_len = compressed_direct_buf_len; + src_len = uncompressed_direct_buf_len; + ret = dlsym_qzCompress(&g_qzCompressSession, uncompressed_bytes, &src_len, + compressed_bytes, &buf_len, 1); + if (ret != QZ_OK){ + THROW(env, "java/lang/InternalError", "Could not compress data, return " + ret); + return 0; + } + if (buf_len > JINT_MAX) { + THROW(env, "java/lang/InternalError", "Invalid return buffer length."); + return 0; + } + + (*env)->SetIntField(env, thisj, QatCompressorJNI_uncompressedDirectBufLen, 0); + return (jint)buf_len; +} + +JNIEXPORT jstring JNICALL +Java_com_intel_qat_jni_QatCompressorJNI_getLibraryName(JNIEnv *env, jclass class) { +#ifdef UNIX + if (dlsym_qzCompress) { + Dl_info dl_info; + if(dladdr( + dlsym_qzCompress, + &dl_info)) { + return (*env)->NewStringUTF(env, dl_info.dli_fname); + } + } + + return (*env)->NewStringUTF(env, LUCENE_QAT_LIBRARY); +#endif + +#ifdef WINDOWS + LPWSTR filename = NULL; + GetLibraryName(dlsym_qzCompress, &filename); + if (filename != NULL) { + return (*env)->NewString(env, filename, (jsize) wcslen(filename)); + } else { + return (*env)->NewStringUTF(env, "Unavailable"); + } +#endif +} + +JNIEXPORT jobject JNICALL +Java_com_intel_qat_jni_QatCompressorJNI_nativeAllocateBB(JNIEnv *env, + jobject obj, jlong capacity, jboolean numa, jboolean force_pinned){ +//void *buf = dlsym_qat_malloc(capacity); +//printf("compressor: DBB address is 0x%lx\n",(unsigned long)buf); +//fflush(stdout); +// return (*env)->NewDirectByteBuffer(env, buf, capacity); + return (*env)->NewDirectByteBuffer(env, dlsym_qzMalloc(capacity, numa, force_pinned), capacity); +} + +#endif //define LUCENE_QAT_LIBRARY diff --git a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c new file mode 100644 index 0000000..8764afe --- /dev/null +++ b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "qatcodec.h" + +#if defined LUCENE_QAT_LIBRARY + +#include +#include +#include +#include + + +#include "com_intel_qat_jni_QatDecompressorJNI.h" + + +static jfieldID QatDecompressorJNI_clazz; +static jfieldID QatDecompressorJNI_compressedDirectBuf; +static jfieldID QatDecompressorJNI_compressedDirectBufLen; +static jfieldID QatDecompressorJNI_uncompressedDirectBuf; +static jfieldID QatDecompressorJNI_directBufferSize; + +#define qaePinnedMemAlloc(x, y) qaeMemAllocNUMA((x), (y), 8) +#define qaePinnedMemFree(x) qaeMemFreeNUMA((void **)&(x)) + +#ifdef UNIX +static int (*dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src, + unsigned int* compressed_buf_len, unsigned char* dest, + unsigned int* uncompressed_buffer_len); +unsigned char* (*dlsym_qzMalloc)(int sz, int numa, int force_pinned); +#endif + +#ifdef WINDOWS +typedef int (__cdecl *__dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src, + unsigned int* compressed_buf_len, unsigned char* dest, + unsigned int* uncompressed_buffer_len); +static __dlsym_qzDecompress dlsym_qzDecompress; +#endif + +JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_initIDs +(JNIEnv *env, jclass clazz){ + + // Load libqatzip.so +#ifdef UNIX + void *libqatzip = dlopen("libqatzip.so", RTLD_LAZY | RTLD_GLOBAL); + if (!libqatzip) { + char msg[128]; + snprintf(msg, sizeof(msg), "%s (%s)!", "Cannot load " LUCENE_QAT_LIBRARY, dlerror()); + THROW(env, "java/lang/UnsatisfiedLinkError", msg); + return; + } +#endif + +#ifdef WINDOWS + HMODULE libqatzip = LoadLibrary(LUCENE_QAT_LIBRARY); + if (!libqatzip) { + THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load qatzip.dll"); + return; + } +#endif + + // Locate the requisite symbols from libqatzip.so +#ifdef UNIX + dlerror(); // Clear any existing error + LOAD_DYNAMIC_SYMBOL(dlsym_qzDecompress, env, libqatzip, "qzDecompress"); + LOAD_DYNAMIC_SYMBOL(dlsym_qzMalloc, env, libqatzip, "qzMalloc"); + +#endif + +#ifdef WINDOWS + LOAD_DYNAMIC_SYMBOL(__dlsym_qatzip_uncompress, dlsym_qzDecompress, env, libqatzip, "qzDecompress"); +#endif + + QatDecompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz", + "Ljava/lang/Class;"); + QatDecompressorJNI_compressedDirectBuf = (*env)->GetFieldID(env,clazz, + "compressedDirectBuf", + "Ljava/nio/Buffer;"); + QatDecompressorJNI_compressedDirectBufLen = (*env)->GetFieldID(env,clazz, + "compressedDirectBufLen", "I"); + QatDecompressorJNI_uncompressedDirectBuf = (*env)->GetFieldID(env,clazz, + "uncompressedDirectBuf", + "Ljava/nio/Buffer;"); + QatDecompressorJNI_directBufferSize = (*env)->GetFieldID(env, clazz, + "directBufferSize", "I"); +} + +JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytesDirect +(JNIEnv *env, jobject thisj){ + const unsigned char* compressed_bytes = NULL; + unsigned char* uncompressed_bytes = NULL; + unsigned int compressed_buf_len; + int ret; + // Get members of QatDecompressorJNI + jobject clazz = (*env)->GetStaticObjectField(env,thisj, QatDecompressorJNI_clazz); + jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, QatDecompressorJNI_compressedDirectBuf); + jint compressed_direct_buf_len = (*env)->GetIntField(env,thisj, QatDecompressorJNI_compressedDirectBufLen); + jobject uncompressed_direct_buf = (*env)->GetObjectField(env,thisj, QatDecompressorJNI_uncompressedDirectBuf); + unsigned int uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, QatDecompressorJNI_directBufferSize); + + // Get the input direct buffer + LOCK_CLASS(env, clazz, "QatDecompressorJNI"); + compressed_bytes = (const unsigned char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); + UNLOCK_CLASS(env, clazz, "QatDecompressorJNI"); + + if (compressed_bytes == 0) { + return (jint)0; + } + + // Get the output direct buffer + LOCK_CLASS(env, clazz, "QatDecompressorJNI"); + uncompressed_bytes = (unsigned char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); + UNLOCK_CLASS(env, clazz, "QatDecompressorJNI"); + + if (uncompressed_bytes == 0) { + return (jint)0; + } + + compressed_buf_len = compressed_direct_buf_len; + ret = dlsym_qzDecompress(&g_qzCompressSession, compressed_bytes, &compressed_buf_len, + uncompressed_bytes, &uncompressed_direct_buf_len); + if (ret != QZ_OK) { + THROW(env, "java/lang/InternalError", "Could not decompress data, return " + ret); + } + + (*env)->SetIntField(env, thisj, QatDecompressorJNI_compressedDirectBufLen, 0); + + return (jint)uncompressed_direct_buf_len; +} + +JNIEXPORT jobject JNICALL +Java_com_intel_qat_jni_QatDecompressorJNI_nativeAllocateBB(JNIEnv *env, + jobject obj, jlong capacity, jboolean numa, jboolean force_pinned){ +/*void *buf = dlsym_qzMalloc(capacity,0,1); +if (NULL == buf){ +fprintf(stderr,"decompressor: DBB address is 0x%lx\n",(unsigned long)buf); +fflush(stderr); +} + return (*env)->NewDirectByteBuffer(env, buf, capacity);*/ + return (*env)->NewDirectByteBuffer(env, dlsym_qzMalloc(capacity, numa, force_pinned), capacity); +} + +#endif //define LUCENE_QAT_LIBRARY diff --git a/lucene_qat_wrapper/src/main/native/com_intel_qat.h b/lucene_qat_wrapper/src/main/native/com_intel_qat.h new file mode 100644 index 0000000..d5a8914 --- /dev/null +++ b/lucene_qat_wrapper/src/main/native/com_intel_qat.h @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This file includes some common utilities + * for all native code used in hadoop. + */ + +#if !defined COM_INTEL_QAT_H +#define COM_INTEL_QAT_H + +#if defined(_WIN32) +#undef UNIX +#define WINDOWS +#else +#undef WINDOWS +#define UNIX +#endif + +/* A helper macro to 'throw' a java exception. */ +#define THROW(env, exception_name, message) \ + { \ + jclass ecls = (*env)->FindClass(env, exception_name); \ + if (ecls) { \ + (*env)->ThrowNew(env, ecls, message); \ + (*env)->DeleteLocalRef(env, ecls); \ + } \ + } + +/* Helper macro to return if an exception is pending */ +#define PASS_EXCEPTIONS(env) \ + { \ + if ((*env)->ExceptionCheck(env)) return; \ + } + +#define PASS_EXCEPTIONS_GOTO(env, target) \ + { \ + if ((*env)->ExceptionCheck(env)) goto target; \ + } + +#define PASS_EXCEPTIONS_RET(env, ret) \ + { \ + if ((*env)->ExceptionCheck(env)) return (ret); \ + } + +/** + * Unix definitions + */ +#ifdef UNIX +#include +#include +#include + +/** + * A helper function to dlsym a 'symbol' from a given library-handle. + * + * @param env jni handle to report contingencies. + * @param handle handle to the dlopen'ed library. + * @param symbol symbol to load. + * @return returns the address where the symbol is loaded in memory, + * NULL on error. + */ +static __attribute__ ((unused)) +void *do_dlsym(JNIEnv *env, void *handle, const char *symbol) { + if (!env) { + return NULL; + } + if (!handle || !symbol) { + THROW(env, "java/lang/InternalError", NULL); + return NULL; + } + char *error = NULL; + void *func_ptr = dlsym(handle, symbol); + if ((error = dlerror()) != NULL) { + THROW(env, "java/lang/UnsatisfiedLinkError", symbol); + return NULL; + } + return func_ptr; +} + +/* A helper macro to dlsym the requisite dynamic symbol and bail-out on error. */ +#define LOAD_DYNAMIC_SYMBOL(func_ptr, env, handle, symbol) \ + if ((func_ptr = do_dlsym(env, handle, symbol)) == NULL) { \ + return; \ + } +#endif +// Unix part end + +#define LOCK_CLASS(env, clazz, classname) \ + if ((*env)->MonitorEnter(env, clazz) != 0) { \ + char exception_msg[128]; \ + snprintf(exception_msg, 128, "Failed to lock %s", classname); \ + THROW(env, "java/lang/InternalError", exception_msg); \ + } + +#define UNLOCK_CLASS(env, clazz, classname) \ + if ((*env)->MonitorExit(env, clazz) != 0) { \ + char exception_msg[128]; \ + snprintf(exception_msg, 128, "Failed to unlock %s", classname); \ + THROW(env, "java/lang/InternalError", exception_msg); \ + } + +#define RETRY_ON_EINTR(ret, expr) do { \ + ret = expr; \ +} while ((ret == -1) && (errno == EINTR)); + +#endif + +//vim: sw=2: ts=2: et diff --git a/lucene_qat_wrapper/src/main/native/config.h b/lucene_qat_wrapper/src/main/native/config.h new file mode 100644 index 0000000..5a7c11b --- /dev/null +++ b/lucene_qat_wrapper/src/main/native/config.h @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* config.h doesn't do anything, but it cannot be dropped while we depend on + * org_apache_hadoop.h (included by qatcodec.h), as the latter includes a + * . */ diff --git a/lucene_qat_wrapper/src/main/native/qatcodec.h b/lucene_qat_wrapper/src/main/native/qatcodec.h new file mode 100644 index 0000000..4c8d1b4 --- /dev/null +++ b/lucene_qat_wrapper/src/main/native/qatcodec.h @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* we currently use a number of macros defined in + * com_intel_qat.h. These are: + * * UNIX + * * THROW + * * LOAD_DYNAMIC_SYMBOL + * * LOCK_CLASS + * * UNLOCK_CLASS + * * Probably at least one of the Windows-specific definitions too + * + * com_intel_qat.h also prevents us dropping config.h, as this file is + * included by com_intel_qat.h. */ +//#include "org_apache_hadoop.h" + +#include "com_intel_qat.h" +#ifdef UNIX +#include +#endif + +#include +#include +#include + +#define LUCENE_QAT_LIBRARY "libqatzip.so" + +extern __thread QzSession_T g_qzCompressSession; diff --git a/lucene_qat_wrapper/src/main/native/util.c b/lucene_qat_wrapper/src/main/native/util.c new file mode 100644 index 0000000..9f53e0e --- /dev/null +++ b/lucene_qat_wrapper/src/main/native/util.c @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +/* +#include "org_apache_hadoop_util_QatNativeCodeLoaderTest.h" +#include "com_intel_qat_util_QatNativeCodeLoader.h" +*/ + +/* + * Class: com_intel_qat_util_QatNativeCodeLoaderTest + * Method: buildSupportsQat + * Signature: ()Z + */ +JNIEXPORT jboolean JNICALL Java_com_intel_qat_util_QatNativeCodeLoaderTest_buildSupportsQat + (JNIEnv *env, jclass cl){ + return 1; + } + +/* + * Class: com_intel_qat_util_QatNativeCodeLoaderTest + * Method: getLibraryName + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_com_intel_qat_util_QatNativeCodeLoaderTest_getLibraryName + (JNIEnv *env, jclass cl) { + jstring name; +// mystring = (*env)->NewStringUTF(env,"QAT Compressor"); + name = (*env)->NewStringUTF(env, "kettle"); + return(name); +} + +/* + * Class: com_intel_qat_util_QatNativeCodeLoader + * Method: buildSupportsQat + * Signature: ()Z + */ +JNIEXPORT jboolean JNICALL Java_com_intel_qat_util_QatNativeCodeLoader_buildSupportsQat + (JNIEnv *env, jclass cl){ + return 1; +} + +/* + * Class: com_intel_qat_util_QatNativeCodeLoader + * Method: getLibraryName + * Signature: ()Ljava/lang/String; + */ +JNIEXPORT jstring JNICALL Java_com_intel_qat_util_QatNativeCodeLoader_getLibraryName + (JNIEnv *env, jclass cl) { + jstring name; +// mystring = (*env)->NewStringUTF(env,"QAT Compressor"); + name = (*env)->NewStringUTF(env, "kettle"); + return(name); +} diff --git a/pom.xml b/pom.xml index 1264f60..19d61c4 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,7 @@ hadoop_qat_wrapper spark_qat_wrapper carbondata_qat_wrapper + lucene_qat_wrapper From 190e8ad43bc87bb411f0e394d34a67825f34d51f Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Tue, 24 Dec 2019 11:11:59 +0800 Subject: [PATCH 02/13] disable the log and change the end in Decompressor --- .../com/intel/qat/jni/QatCompressorJNI.java | 35 ++++++++++--------- .../com/intel/qat/jni/QatDecompressorJNI.java | 28 ++++++++------- .../intel/qat/util/QatNativeCodeLoader.java | 27 +++++++------- lucene_qat_wrapper/src/main/native/util.c | 1 - 4 files changed, 49 insertions(+), 42 deletions(-) diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java index f877f2d..1125334 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java @@ -22,11 +22,11 @@ import java.io.IOException; import java.nio.Buffer; import java.nio.ByteBuffer; -//import org.apache.logging.log4j.LogManager; -//import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; import com.intel.qat.conf.QatConfigurationKeys; import com.intel.qat.util.QatNativeCodeLoader; @@ -41,8 +41,8 @@ public class QatCompressorJNI { - private static final Logger LOG = LoggerFactory.getLogger(QatCompressorJNI.class.getName()); - //private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); + //private static final Logger LOG = LoggerFactory.getLogger(QatCompressorJNI.class.getName()); + // private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; // HACK - Use this as a global lock in the JNI layer @@ -65,6 +65,7 @@ public class QatCompressorJNI { static { if (QatNativeCodeLoader.isNativeCodeLoaded() && QatNativeCodeLoader.buildSupportsQat()) { + System.out.println("-------->the library name is " + QatNativeCodeLoader.getLibraryName()); try { String value = System.getProperty("QAT_COMPRESS_LEVEL"); int level = 1; @@ -73,21 +74,21 @@ public class QatCompressorJNI { level = Integer.parseInt(value); if (level < 1 || level > 9) { level = 1; - LOG.warn("Invalid value for compression level:" + value + /* LOG.warn("Invalid value for compression level:" + value + ", value should be in range 1-9." - + " Proceeding with default value as 1."); + + " Proceeding with default value as 1.");*/ } } catch (NumberFormatException e) { level = 1; - LOG.warn("Could not parse the value:" + value + /* LOG.warn("Could not parse the value:" + value + ", compression level should be in range 1-9." - + " Proceeding with default value as 1."); + + " Proceeding with default value as 1.");*/ } } initIDs(level); nativeQatLoaded = true; } catch (Throwable t) { - LOG.error("failed to load QatCompressor AMAC QatCompressor", t); + // LOG.error("failed to load QatCompressor AMAC QatCompressor", t); } } } @@ -107,23 +108,23 @@ public QatCompressorJNI(int directBufferSize, boolean useNativeAllocateBB, boolean forcePinned, boolean numa) { this.directBufferSize = directBufferSize; if (useNativeAllocateBB) { - LOG.info("Creating ByteBuffer's using nativeAllocateBB."); + // LOG.info("Creating ByteBuffer's using nativeAllocateBB."); try { uncompressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, numa, forcePinned); } catch (Throwable t) { - LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + /* LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + " for uncompressed direct ByteBuffer. Creating the uncompressed" - + " ByteBuffer using ByteBuffer.allocateDirect().", t); + + " ByteBuffer using ByteBuffer.allocateDirect().", t);*/ uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); } try { compressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, numa, forcePinned); } catch (Throwable t) { - LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + /* LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + " for compressed direct ByteBuffer. Creating the compressed" - + " ByteBuffer using ByteBuffer.allocateDirect().", t); + + " ByteBuffer using ByteBuffer.allocateDirect().", t);*/ compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); } } else { @@ -354,4 +355,4 @@ public synchronized void end() { public native Object nativeAllocateBB(long capacity, boolean numa, boolean forcePinned); -} \ No newline at end of file +} diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java index e7fcf7c..f1a0a04 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -24,11 +24,11 @@ import com.intel.qat.conf.QatConfigurationKeys; import com.intel.qat.util.QatNativeCodeLoader; -//import org.apache.logging.log4j.LogManager; -//import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; /** * @author root @@ -37,8 +37,8 @@ * @link qat-parent */ public class QatDecompressorJNI { - private static final Logger LOG = LoggerFactory.getLogger(QatDecompressorJNI.class.getName()); - //private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); + // private static final Logger LOG = LoggerFactory.getLogger(QatDecompressorJNI.class.getName()); + // private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; @@ -63,7 +63,8 @@ public class QatDecompressorJNI { initIDs(); nativeQatLoaded = true; } catch (Throwable t) { - LOG.error("failed to load QatDecompressor", t); + // LOG.error("failed to load QatDecompressor", t); + System.out.println("failed to load QatDecompressor" + t); } } } @@ -85,23 +86,23 @@ public QatDecompressorJNI(int directBufferSize, boolean useNativeAllocateBB, boolean forcePinned, boolean numa) { this.directBufferSize = directBufferSize; if (useNativeAllocateBB) { - LOG.info("Creating ByteBuffer's using nativeAllocateBB."); + // LOG.info("Creating ByteBuffer's using nativeAllocateBB."); try { uncompressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, numa, forcePinned); } catch (Throwable t) { - LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + /* LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + " for uncompressed direct ByteBuffer. Creating the uncompressed" - + " ByteBuffer using ByteBuffer.allocateDirect().", t); + + " ByteBuffer using ByteBuffer.allocateDirect().", t);*/ uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); } try { compressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, numa, forcePinned); } catch (Throwable t) { - LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + /* LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + " for compressed direct ByteBuffer. Creating the compressed" - + " ByteBuffer using ByteBuffer.allocateDirect().", t); + + " ByteBuffer using ByteBuffer.allocateDirect().", t);*/ compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); } } else { @@ -319,6 +320,9 @@ public void reset() { */ public void end() { // do nothing + if(finished()){ + reset(); + } } private native static void initIDs(); diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java index 502b3cb..6f78c39 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java @@ -18,11 +18,11 @@ package com.intel.qat.util; -//import org.apache.logging.log4j.LogManager; -//import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; /** * A helper to load the native qat code i.e. libqat.so. * This handles the fallback to either the bundled libhadoop-Linux-i386-32.so @@ -31,31 +31,34 @@ */ public class QatNativeCodeLoader { - private static final Logger LOG = LoggerFactory.getLogger(QatNativeCodeLoader.class); + //private static final Logger LOG = LoggerFactory.getLogger(QatNativeCodeLoader.class); //private static final Logger LOG = LogManager.getLogger(QatNativeCodeLoader.class); private static boolean nativeCodeLoaded = false; static { // Try to load native qat library and set fallback flag appropriately - if(LOG.isDebugEnabled()) { + /* if(LOG.isDebugEnabled()) { LOG.debug("Trying to load the custom-built native-qat library..."); - } + }*/ try { System.loadLibrary("qatcodec"); - LOG.debug("Loaded the native-qat library"); + //LOG.debug("Loaded the native-qat library"); nativeCodeLoaded = true; } catch (Throwable t) { // Ignore failure to load - if(LOG.isDebugEnabled()) { + System.out.println("Failed to load native-qat with error: " + t); + System.out.println("java.library.path=" + + System.getProperty("java.library.path")); + /*if(LOG.isDebugEnabled()) { LOG.debug("Failed to load native-qat with error: " + t); LOG.debug("java.library.path=" + System.getProperty("java.library.path")); - } + }*/ } if (!nativeCodeLoaded) { - LOG.warn("Unable to load native-qat library for your platform... " + - "using builtin-java classes where applicable"); + /* LOG.warn("Unable to load native-qat library for your platform... " + + "using builtin-java classes where applicable");*/ } } diff --git a/lucene_qat_wrapper/src/main/native/util.c b/lucene_qat_wrapper/src/main/native/util.c index 9f53e0e..17fcc11 100644 --- a/lucene_qat_wrapper/src/main/native/util.c +++ b/lucene_qat_wrapper/src/main/native/util.c @@ -23,7 +23,6 @@ #include "org_apache_hadoop_util_QatNativeCodeLoaderTest.h" #include "com_intel_qat_util_QatNativeCodeLoader.h" */ - /* * Class: com_intel_qat_util_QatNativeCodeLoaderTest * Method: buildSupportsQat From a564be22323f4f055dd009fd31b0f184f59ae370 Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Tue, 24 Dec 2019 12:20:34 +0800 Subject: [PATCH 03/13] enable log --- .../hadoop/util/.QatNativeCodeLoader.java.swp | Bin 0 -> 12288 bytes lucene_qat_wrapper/pom.xml | 6 ++++ .../com/intel/qat/jni/QatCompressorJNI.java | 29 ++++++++---------- .../com/intel/qat/jni/QatDecompressorJNI.java | 27 +++++++--------- .../intel/qat/util/QatNativeCodeLoader.java | 26 ++++++++-------- 5 files changed, 42 insertions(+), 46 deletions(-) create mode 100644 hadoop_qat_wrapper/src/main/java/org/apache/hadoop/util/.QatNativeCodeLoader.java.swp diff --git a/hadoop_qat_wrapper/src/main/java/org/apache/hadoop/util/.QatNativeCodeLoader.java.swp b/hadoop_qat_wrapper/src/main/java/org/apache/hadoop/util/.QatNativeCodeLoader.java.swp new file mode 100644 index 0000000000000000000000000000000000000000..5b0952e7a5fc5ffac75f4876ba45df50e34f4328 GIT binary patch literal 12288 zcmeHN&yO5O6)s0WAR!QuGtu)P!Fbv2c5Evlb_^Np-7sRAwLPRJzbsZ zYL7=zNW_5)7eGi{`3pFJ1UVsr#EDxjNe($dA|Oayh;oSp->dH4-K>W|j!~+mZ@QiziMt5>tyH$8iE_$_)Yx*%{pD1XdF*|z%3gTn@%I_; zhT=ux;;YwgrMfye7e6(e#KXd={Q6*W#iS~U=Q1^>y!EVfx0Y6xrLu8llXxL@5r0qK zma(x}EK8ZpRXaDXozCOy(v76kw-t-oH8NE;;`yE4a1RUw2AF{d#QAfNoaLu4^iI=5 zpWElha0Lbe1A&3SKwuy+5Euvy1O@^F9}xyzc}lzhU+-!Cz24qWow&DO!!0ln7zhjm z1_A?tfxtjuATSUZ2n+-U0t1184KDNRp2Gyx4@5qDexuWt^0-eJMbp(OW?b} z7lG5j=Ye~GH|`VS55Vt%UjPNL04@TL0}=3-PYLly;5ncM4uEaoA>d!1gnr;9;OD?i zU<-KXULpPo{0;ak@F(C!U;tbK9s?c)&H`_L0&{`i06zhK1biF#8gL1?2wVXE0q))e zehvHz_yNFtz5s+PFc26B4E#3+o)<)AJ2HU~L6Im~@wfnFID`lnA2Uoy5g{L5S%WyxDAbxTvnB;XgOC_5gOYN zZ8^QCm@EE`^r)}CaEF8e*LmXUuD}$;AIff==j}{gUdbY*N;xo@ z=TnKTG0SDf*@~pFTB1>N1y~mRal9-*SCpG9 zL&8T#8S#J4Mkb*P_&{bj!S3>gTY@>&_$=7~2@BnY2mH2F@9IPqmGa>kD^v||*uHCv zKC-%036ny(EV7z)W#+cB1#eOCztC0XgO0$mqNj(GZ|v<)=;^_Y8-vkgxIL!58+2uF zbaglx?v3zmhX$i(=$pgQ)y*#K>P~4@RdI50E@_o|lh7cI(|=k!WW!xX^Z9gqf++~6 zes3_Q;kZX%AB>0NO*Ux1K)zKtou93>g`nx`*wF*0maJ7~R$&Z@s0&(JjWa9Tq{_@3 zjw1yzEQznVbLGW2UM`ni@^CXY4l?Gkig$-swnyXbt#i@Yc9E~@$9Rp1o-s8oRl9>HAdLpv|BDw8(l=(1T>XLIb8Wr3MM6 mn}BRj0IzR}@-bu{TUW1.7.2 test + + log4j + apache-log4j-extras + 1.2.17 + compile + diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java index 1125334..dbd8829 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java @@ -22,11 +22,9 @@ import java.io.IOException; import java.nio.Buffer; import java.nio.ByteBuffer; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; import com.intel.qat.conf.QatConfigurationKeys; import com.intel.qat.util.QatNativeCodeLoader; @@ -41,8 +39,7 @@ public class QatCompressorJNI { - //private static final Logger LOG = LoggerFactory.getLogger(QatCompressorJNI.class.getName()); - // private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); + private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; // HACK - Use this as a global lock in the JNI layer @@ -74,21 +71,21 @@ public class QatCompressorJNI { level = Integer.parseInt(value); if (level < 1 || level > 9) { level = 1; - /* LOG.warn("Invalid value for compression level:" + value + LOG.warn("Invalid value for compression level:" + value + ", value should be in range 1-9." - + " Proceeding with default value as 1.");*/ + + " Proceeding with default value as 1."); } } catch (NumberFormatException e) { level = 1; - /* LOG.warn("Could not parse the value:" + value + LOG.warn("Could not parse the value:" + value + ", compression level should be in range 1-9." - + " Proceeding with default value as 1.");*/ + + " Proceeding with default value as 1."); } } initIDs(level); nativeQatLoaded = true; } catch (Throwable t) { - // LOG.error("failed to load QatCompressor AMAC QatCompressor", t); + LOG.error("failed to load QatCompressor AMAC QatCompressor", t); } } } @@ -108,23 +105,23 @@ public QatCompressorJNI(int directBufferSize, boolean useNativeAllocateBB, boolean forcePinned, boolean numa) { this.directBufferSize = directBufferSize; if (useNativeAllocateBB) { - // LOG.info("Creating ByteBuffer's using nativeAllocateBB."); + LOG.info("Creating ByteBuffer's using nativeAllocateBB."); try { uncompressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, numa, forcePinned); } catch (Throwable t) { - /* LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + " for uncompressed direct ByteBuffer. Creating the uncompressed" - + " ByteBuffer using ByteBuffer.allocateDirect().", t);*/ + + " ByteBuffer using ByteBuffer.allocateDirect().", t); uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); } try { compressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, numa, forcePinned); } catch (Throwable t) { - /* LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + " for compressed direct ByteBuffer. Creating the compressed" - + " ByteBuffer using ByteBuffer.allocateDirect().", t);*/ + + " ByteBuffer using ByteBuffer.allocateDirect().", t); compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); } } else { diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java index f1a0a04..29693a7 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -24,11 +24,8 @@ import com.intel.qat.conf.QatConfigurationKeys; import com.intel.qat.util.QatNativeCodeLoader; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; /** * @author root @@ -37,8 +34,7 @@ * @link qat-parent */ public class QatDecompressorJNI { - // private static final Logger LOG = LoggerFactory.getLogger(QatDecompressorJNI.class.getName()); - // private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); + private static final Logger LOG = LogManager.getLogger(QatDecompressorJNI.class.getName()); private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; @@ -63,8 +59,7 @@ public class QatDecompressorJNI { initIDs(); nativeQatLoaded = true; } catch (Throwable t) { - // LOG.error("failed to load QatDecompressor", t); - System.out.println("failed to load QatDecompressor" + t); + LOG.error("failed to load QatDecompressor", t); } } } @@ -86,23 +81,23 @@ public QatDecompressorJNI(int directBufferSize, boolean useNativeAllocateBB, boolean forcePinned, boolean numa) { this.directBufferSize = directBufferSize; if (useNativeAllocateBB) { - // LOG.info("Creating ByteBuffer's using nativeAllocateBB."); + LOG.info("Creating ByteBuffer's using nativeAllocateBB."); try { uncompressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, numa, forcePinned); } catch (Throwable t) { - /* LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + " for uncompressed direct ByteBuffer. Creating the uncompressed" - + " ByteBuffer using ByteBuffer.allocateDirect().", t);*/ + + " ByteBuffer using ByteBuffer.allocateDirect().", t); uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); } try { compressedDirectBuf = (ByteBuffer) nativeAllocateBB(directBufferSize, numa, forcePinned); } catch (Throwable t) { - /* LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + LOG.error("Failed to create ByteBuffer using nativeAllocateBB" + " for compressed direct ByteBuffer. Creating the compressed" - + " ByteBuffer using ByteBuffer.allocateDirect().", t);*/ + + " ByteBuffer using ByteBuffer.allocateDirect().", t); compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); } } else { @@ -320,9 +315,9 @@ public void reset() { */ public void end() { // do nothing - if(finished()){ + /* if(finished()){ reset(); - } + }*/ } private native static void initIDs(); diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java index 6f78c39..f8c71c2 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java @@ -18,9 +18,10 @@ package com.intel.qat.util; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +//import org.apache.logging.log4j.LogManager; +//import org.apache.logging.log4j.Logger; //import org.slf4j.Logger; //import org.slf4j.LoggerFactory; /** @@ -32,33 +33,30 @@ public class QatNativeCodeLoader { //private static final Logger LOG = LoggerFactory.getLogger(QatNativeCodeLoader.class); - //private static final Logger LOG = LogManager.getLogger(QatNativeCodeLoader.class); + private static final Logger LOG = LogManager.getLogger(QatNativeCodeLoader.class); private static boolean nativeCodeLoaded = false; static { // Try to load native qat library and set fallback flag appropriately - /* if(LOG.isDebugEnabled()) { + if(LOG.isDebugEnabled()) { LOG.debug("Trying to load the custom-built native-qat library..."); - }*/ + } try { System.loadLibrary("qatcodec"); - //LOG.debug("Loaded the native-qat library"); + LOG.debug("Loaded the native-qat library"); nativeCodeLoaded = true; } catch (Throwable t) { // Ignore failure to load - System.out.println("Failed to load native-qat with error: " + t); - System.out.println("java.library.path=" + - System.getProperty("java.library.path")); - /*if(LOG.isDebugEnabled()) { + if(LOG.isDebugEnabled()) { LOG.debug("Failed to load native-qat with error: " + t); LOG.debug("java.library.path=" + System.getProperty("java.library.path")); - }*/ + } } if (!nativeCodeLoaded) { - /* LOG.warn("Unable to load native-qat library for your platform... " + - "using builtin-java classes where applicable");*/ + LOG.warn("Unable to load native-qat library for your platform... " + + "using builtin-java classes where applicable"); } } From de17216b1dce293993ad050a590eef8c465ef523 Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Fri, 3 Jan 2020 13:05:44 +0800 Subject: [PATCH 04/13] add the log file to test the content in compressed bytes --- .../hadoop/util/.QatNativeCodeLoader.java.swp | Bin 12288 -> 0 bytes lucene_qat_wrapper/pom.xml | 60 ++++++++++++++ .../com/intel/qat/jni/QatDecompressorJNI.java | 29 +++++++ .../intel/qat/jni/QatDecompressorJNITest.java | 44 +++++++++++ .../src/main/native/QatCompressorJNI.c | 2 +- .../src/main/native/QatDecompressorJNI.c | 73 ++++++++++++++++++ 6 files changed, 207 insertions(+), 1 deletion(-) delete mode 100644 hadoop_qat_wrapper/src/main/java/org/apache/hadoop/util/.QatNativeCodeLoader.java.swp create mode 100644 lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNITest.java diff --git a/hadoop_qat_wrapper/src/main/java/org/apache/hadoop/util/.QatNativeCodeLoader.java.swp b/hadoop_qat_wrapper/src/main/java/org/apache/hadoop/util/.QatNativeCodeLoader.java.swp deleted file mode 100644 index 5b0952e7a5fc5ffac75f4876ba45df50e34f4328..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHN&yO5O6)s0WAR!QuGtu)P!Fbv2c5Evlb_^Np-7sRAwLPRJzbsZ zYL7=zNW_5)7eGi{`3pFJ1UVsr#EDxjNe($dA|Oayh;oSp->dH4-K>W|j!~+mZ@QiziMt5>tyH$8iE_$_)Yx*%{pD1XdF*|z%3gTn@%I_; zhT=ux;;YwgrMfye7e6(e#KXd={Q6*W#iS~U=Q1^>y!EVfx0Y6xrLu8llXxL@5r0qK zma(x}EK8ZpRXaDXozCOy(v76kw-t-oH8NE;;`yE4a1RUw2AF{d#QAfNoaLu4^iI=5 zpWElha0Lbe1A&3SKwuy+5Euvy1O@^F9}xyzc}lzhU+-!Cz24qWow&DO!!0ln7zhjm z1_A?tfxtjuATSUZ2n+-U0t1184KDNRp2Gyx4@5qDexuWt^0-eJMbp(OW?b} z7lG5j=Ye~GH|`VS55Vt%UjPNL04@TL0}=3-PYLly;5ncM4uEaoA>d!1gnr;9;OD?i zU<-KXULpPo{0;ak@F(C!U;tbK9s?c)&H`_L0&{`i06zhK1biF#8gL1?2wVXE0q))e zehvHz_yNFtz5s+PFc26B4E#3+o)<)AJ2HU~L6Im~@wfnFID`lnA2Uoy5g{L5S%WyxDAbxTvnB;XgOC_5gOYN zZ8^QCm@EE`^r)}CaEF8e*LmXUuD}$;AIff==j}{gUdbY*N;xo@ z=TnKTG0SDf*@~pFTB1>N1y~mRal9-*SCpG9 zL&8T#8S#J4Mkb*P_&{bj!S3>gTY@>&_$=7~2@BnY2mH2F@9IPqmGa>kD^v||*uHCv zKC-%036ny(EV7z)W#+cB1#eOCztC0XgO0$mqNj(GZ|v<)=;^_Y8-vkgxIL!58+2uF zbaglx?v3zmhX$i(=$pgQ)y*#K>P~4@RdI50E@_o|lh7cI(|=k!WW!xX^Z9gqf++~6 zes3_Q;kZX%AB>0NO*Ux1K)zKtou93>g`nx`*wF*0maJ7~R$&Z@s0&(JjWa9Tq{_@3 zjw1yzEQznVbLGW2UM`ni@^CXY4l?Gkig$-swnyXbt#i@Yc9E~@$9Rp1o-s8oRl9>HAdLpv|BDw8(l=(1T>XLIb8Wr3MM6 mn}BRj0IzR}@-bu{TUWcompile + + org.junit.jupiter + junit-jupiter-engine + 5.2.0 + test + + + + org.junit.jupiter + junit-jupiter + RELEASE + compile + + + org.junit.jupiter + junit-jupiter + RELEASE + test + + + org.testng + testng + RELEASE + compile + + + junit + junit + 4.12 + compile + + + junit + junit + 4.12 + test + + + org.junit.jupiter + junit-jupiter-api + 5.6.0-M1 + compile + + + + + org.junit.platform + junit-platform-launcher + 1.6.0-M1 + test + + + + org.junit.vintage + junit-vintage-engine + 5.6.0-M1 + test + + + diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java index 29693a7..d97dc16 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.nio.Buffer; import java.nio.ByteBuffer; +import java.util.Arrays; import com.intel.qat.conf.QatConfigurationKeys; import com.intel.qat.util.QatNativeCodeLoader; @@ -158,6 +159,12 @@ public void setInput(byte[] b, int off, int len) { // Reinitialize qat's output direct-buffer uncompressedDirectBuf.limit(directBufferSize); uncompressedDirectBuf.position(directBufferSize); + + System.out.println("-------------> the result in the setInput is :<-------------------"); + System.out.println("the content in the userBuf is: " + Arrays.toString(this.userBuf)); + System.out.println("the userBufOff is : " + this.userBufOff); + System.out.println("the userBufLen is : " + this.userBufLen); + System.out.println("----------------end of the setInput ---------------------"); } /** @@ -252,6 +259,10 @@ public boolean finished() { public int decompress(byte[] b, int off, int len) throws IOException { + + System.out.println("------------go into the decompress ----------------"); + System.out.println("the byte[] need to be decompressed is : " + Arrays.toString(b)); + System.out.println("the content in compressedBuffer is : "+ compressedDirectBuf); if (b == null) { throw new NullPointerException(); } @@ -273,8 +284,23 @@ public int decompress(byte[] b, int off, int len) uncompressedDirectBuf.rewind(); uncompressedDirectBuf.limit(directBufferSize); + System.out.println("the capacity of compressedDirectBuf is : " + this.compressedDirectBuf.capacity()); + System.out.println("the position of compressedDirectBuf is : " + this.compressedDirectBuf.position()); + System.out.println("the limit of the compressedDirectBuf is : " + this.compressedDirectBuf.limit()); + + + System.out.println("the capacity of uncompressedDirectBuf is : " + this.uncompressedDirectBuf.capacity()); + System.out.println("the position of uncompressedDirectBuf is : " + this.uncompressedDirectBuf.position()); + System.out.println("the limit of the uncompressedDirectBuf is : " + this.uncompressedDirectBuf.limit()); + System.out.println("the off of userBuf is :" + this.userBufOff); + System.out.println("the len of the userBuf is : " + this.userBufLen); + System.out.println("----------->next step is using the native function :"); + // Decompress data n = decompressBytesDirect(); + + System.out.println("----------------end of using native func"); + uncompressedDirectBuf.limit(n); if (userBufLen <= 0) { @@ -284,8 +310,11 @@ public int decompress(byte[] b, int off, int len) // Get atmost 'len' bytes n = Math.min(n, len); ((ByteBuffer) uncompressedDirectBuf).get(b, off, n); + + System.out.println("-------------go out of the condition compressedDirectBufLen"); } + System.out.println("-----------------go out of the decompress --------------"); return n; } diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNITest.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNITest.java new file mode 100644 index 0000000..586b35c --- /dev/null +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNITest.java @@ -0,0 +1,44 @@ +package com.intel.qat.jni; + +import org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; + + +import java.io.IOException; + +class QatDecompressorJNITest { + + //@org.junit.jupiter.api.Test + @Test + void isNativeCodeLoaded() { + } + + @Test + void setInput() { + } + + @Test + void decompress() throws IOException { + for(int i = 0; i < 3; i++){ + final byte[] decompressed = new byte[]{1,2,3,4}; + final int off = 1; + final int len = 3; + final byte[] compressed = new byte[]{37,31,-117,8,4,0,0,0,0,0,-1,12,0,81,90,8,0,0,0,0,0,0,0,0,0,99,98,102,1,0,-90,-102,-123,-48,3,0,0,0}; + QatDecompressorJNI qatDecompressor = new QatDecompressorJNI(); + qatDecompressor.reset(); + qatDecompressor.setInput(compressed, 0,38 ); + byte[] bytes = new byte[3]; + int length = qatDecompressor.decompress(bytes,0,3); + System.out.println("the decompressed length is : "+ length); + + } + } + + @Test + void reset() { + } + + @Test + void end() { + } +} \ No newline at end of file diff --git a/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c index f9aa149..114ee5d 100644 --- a/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c +++ b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c @@ -92,7 +92,7 @@ JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatCompressorJNI_initIDs "directBufferSize", "I"); dlsym_qzGetDefaults(¶ms); params.comp_lvl = level; - fprintf(stderr, "compression level is %d, tid is %d\n", level, syscall(__NR_gettid)); + fprintf(stderr, "compression level is eeeeeeeee %d, tid is %d\n", level, syscall(__NR_gettid)); dlsym_qzSetDefaults(¶ms); } diff --git a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c index 8764afe..eb2c40c 100644 --- a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c +++ b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c @@ -86,6 +86,9 @@ JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_initIDs LOAD_DYNAMIC_SYMBOL(__dlsym_qatzip_uncompress, dlsym_qzDecompress, env, libqatzip, "qzDecompress"); #endif + + fprintf(stderr, "-------> decompression level is eeeeeeeee\n"); + fflush(stderr); QatDecompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz", "Ljava/lang/Class;"); QatDecompressorJNI_compressedDirectBuf = (*env)->GetFieldID(env,clazz, @@ -106,6 +109,22 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes unsigned char* uncompressed_bytes = NULL; unsigned int compressed_buf_len; int ret; + + FILE *p; + //p = fopen("/home/sparkuser/Downloads/QatDecompressorJNI_decompressBytesDirect.txt","wb+"); + //p = fopen("/home/sparkuser/Downloads/QatDecompressorJNI_decompressBytesDirect.txt","w+"); + p = fopen("/tmp/QatDecompressorJNI_decompressBytesDirect1.txt","a+"); + if(p!=NULL){ + fprintf(stderr,"the file have open \n"); + fflush(stderr); + } + else{ + fprintf(stderr,"the file not open \n"); + fflush(stderr); + } + + fprintf(stderr,"----->out of file \n"); + // Get members of QatDecompressorJNI jobject clazz = (*env)->GetStaticObjectField(env,thisj, QatDecompressorJNI_clazz); jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, QatDecompressorJNI_compressedDirectBuf); @@ -113,11 +132,26 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes jobject uncompressed_direct_buf = (*env)->GetObjectField(env,thisj, QatDecompressorJNI_uncompressedDirectBuf); unsigned int uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, QatDecompressorJNI_directBufferSize); + + fprintf(p,"Before decompress, the compressed_direct_buf_len is %d \n",compressed_direct_buf_len); + fprintf(p,"Before decompress, the uncompressed_direct_buf_len is %d \n",uncompressed_direct_buf_len); + + + // Get the input direct buffer LOCK_CLASS(env, clazz, "QatDecompressorJNI"); compressed_bytes = (const unsigned char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); UNLOCK_CLASS(env, clazz, "QatDecompressorJNI"); + fprintf(p,"the compressed_bytes get from the input direct buffer : \n"); + // fputs(compressed_bytes,p); + + int i; + for(i = 0; i < compressed_buf_len; i++){ + fprintf(p,"%02x , ",compressed_bytes[i]); + fprintf(stderr,"%02x , ",compressed_bytes[i]); + } + if (compressed_bytes == 0) { return (jint)0; } @@ -127,13 +161,52 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes uncompressed_bytes = (unsigned char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); UNLOCK_CLASS(env, clazz, "QatDecompressorJNI"); + fprintf(p,"the uncompressed_bytes get from the input direct buffer : \n"); + //fputs(uncompressed_bytes,p); + //fprintf(p,"\n"); + if (uncompressed_bytes == 0) { return (jint)0; } + + for(i = 0; i < uncompressed_direct_buf_len; i++){ + fprintf(p,"%02x , ",uncompressed_bytes[i]); + fprintf(stderr,"%02x , ",uncompressed_bytes[i]); + } + + compressed_buf_len = compressed_direct_buf_len; ret = dlsym_qzDecompress(&g_qzCompressSession, compressed_bytes, &compressed_buf_len, uncompressed_bytes, &uncompressed_direct_buf_len); + + + fprintf(p,"\n the compressed_buf_len after qzDecompress %d: \n",&compressed_buf_len); + fprintf(p,"\n the uncompressed_direct_buf_len after qzDecompress %d : \n", &uncompressed_direct_buf_len); + + + fprintf(p, "\n the compressed_bytes after qzDecompress : \n"); + //fputs(compressed_bytes,p); + for(i = 0; i < compressed_buf_len; i++){ + fprintf(p,"%02x , ",compressed_bytes[i]); + fprintf(stderr,"%02x , ",compressed_bytes[i]); + } + + fprintf(p,"\n the uncompressed_bytes after qaDecompress : \n"); + //fputs(uncompressed_bytes,p); + for(i = 0; i < uncompressed_direct_buf_len; i++){ + fprintf(p,"%02x , ",uncompressed_bytes[i]); + fprintf(stderr,"%02x , ",uncompressed_bytes[i]); + } + + fprintf(p,"\n the result of the qzDecompress is %d ", ret); + + fprintf(stderr,"\n the result of the qzDecompress is %d ", ret); + + // fwrite(in,srcLen,1,p); + //fprintf(p,"%d",out); + fclose(p); + if (ret != QZ_OK) { THROW(env, "java/lang/InternalError", "Could not decompress data, return " + ret); } From e084ce82410f65450158654ecd0b229500dbdf53 Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Tue, 7 Jan 2020 17:56:16 +0800 Subject: [PATCH 05/13] modified the QatDecompressorJNI.c to enable the twice output to the file --- .../src/main/native/QatDecompressorJNI.c | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c index eb2c40c..45d7e7e 100644 --- a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c +++ b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c @@ -38,6 +38,10 @@ static jfieldID QatDecompressorJNI_directBufferSize; #define qaePinnedMemAlloc(x, y) qaeMemAllocNUMA((x), (y), 8) #define qaePinnedMemFree(x) qaeMemFreeNUMA((void **)&(x)) +__thread QzSession_T g_qzDecompressSession = { + .internal = NULL, +}; + #ifdef UNIX static int (*dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src, unsigned int* compressed_buf_len, unsigned char* dest, @@ -87,7 +91,7 @@ JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_initIDs #endif - fprintf(stderr, "-------> decompression level is eeeeeeeee\n"); + fprintf(stderr, "-------> decompression level is feeeeeeeffffffff\n"); fflush(stderr); QatDecompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz", "Ljava/lang/Class;"); @@ -114,6 +118,8 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes //p = fopen("/home/sparkuser/Downloads/QatDecompressorJNI_decompressBytesDirect.txt","wb+"); //p = fopen("/home/sparkuser/Downloads/QatDecompressorJNI_decompressBytesDirect.txt","w+"); p = fopen("/tmp/QatDecompressorJNI_decompressBytesDirect1.txt","a+"); + fprintf(stderr, " tid is %d\n", syscall(__NR_gettid)); + fprintf(p, " tid is %d\n", syscall(__NR_gettid)); if(p!=NULL){ fprintf(stderr,"the file have open \n"); fflush(stderr); @@ -137,7 +143,6 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes fprintf(p,"Before decompress, the uncompressed_direct_buf_len is %d \n",uncompressed_direct_buf_len); - // Get the input direct buffer LOCK_CLASS(env, clazz, "QatDecompressorJNI"); compressed_bytes = (const unsigned char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); @@ -147,12 +152,14 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes // fputs(compressed_bytes,p); int i; - for(i = 0; i < compressed_buf_len; i++){ + for(i = 0; i < compressed_direct_buf_len; i++){ fprintf(p,"%02x , ",compressed_bytes[i]); fprintf(stderr,"%02x , ",compressed_bytes[i]); } if (compressed_bytes == 0) { + fprintf(p,"---------->, the compressed_bytes is 0 \n",compressed_direct_buf_len); + fprintf(stderr, "-------> the compressed_bytes is 0\n"); return (jint)0; } @@ -161,47 +168,48 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes uncompressed_bytes = (unsigned char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); UNLOCK_CLASS(env, clazz, "QatDecompressorJNI"); - fprintf(p,"the uncompressed_bytes get from the input direct buffer : \n"); + //fprintf(p,"the uncompressed_bytes get from the input direct buffer : \n"); //fputs(uncompressed_bytes,p); //fprintf(p,"\n"); - if (uncompressed_bytes == 0) { - return (jint)0; - } - - - for(i = 0; i < uncompressed_direct_buf_len; i++){ + /* for(i = 0; i < uncompressed_direct_buf_len; i++){ fprintf(p,"%02x , ",uncompressed_bytes[i]); fprintf(stderr,"%02x , ",uncompressed_bytes[i]); - } + }*/ + if (uncompressed_bytes == 0) { + return (jint)0; + } - compressed_buf_len = compressed_direct_buf_len; - ret = dlsym_qzDecompress(&g_qzCompressSession, compressed_bytes, &compressed_buf_len, +const unsigned char compressed_bytes1[38] = {0x2f , 0x8b , 0x08 , 0x04 , 00 , 00 , 00 , 00 , 00 , 0xff , 0x0c , 00 , 0x51 , 0x5a , 0x08 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 0x63 , 0x62 , 0x66 , 01 , 00 , 0xa6 , 0x9a , 0x85 , 0xd0 , 03 , 00 , 00 , 00 , 00}; +compressed_buf_len = 38; + //compressed_buf_len = compressed_direct_buf_len; + ret = dlsym_qzDecompress(&g_qzDecompressSession, compressed_bytes1, &compressed_buf_len, uncompressed_bytes, &uncompressed_direct_buf_len); - fprintf(p,"\n the compressed_buf_len after qzDecompress %d: \n",&compressed_buf_len); - fprintf(p,"\n the uncompressed_direct_buf_len after qzDecompress %d : \n", &uncompressed_direct_buf_len); + fprintf(p,"\n the compressed_buf_len after qzDecompress %d: \n",compressed_buf_len); + fprintf(p,"\n the uncompressed_direct_buf_len after qzDecompress %d : \n", uncompressed_direct_buf_len); fprintf(p, "\n the compressed_bytes after qzDecompress : \n"); //fputs(compressed_bytes,p); - for(i = 0; i < compressed_buf_len; i++){ + for(i = 0; i < compressed_direct_buf_len; i++){ fprintf(p,"%02x , ",compressed_bytes[i]); fprintf(stderr,"%02x , ",compressed_bytes[i]); } - fprintf(p,"\n the uncompressed_bytes after qaDecompress : \n"); + /*fprintf(p,"\n the uncompressed_bytes after qaDecompress : \n"); //fputs(uncompressed_bytes,p); for(i = 0; i < uncompressed_direct_buf_len; i++){ fprintf(p,"%02x , ",uncompressed_bytes[i]); fprintf(stderr,"%02x , ",uncompressed_bytes[i]); - } + }*/ + fprintf(p,"\n the result of the qzDecompress is %d ", ret); - fprintf(stderr,"\n the result of the qzDecompress is %d ", ret); + fprintf(stderr,"\n the result of the qzDecompress is %d \n", ret); // fwrite(in,srcLen,1,p); //fprintf(p,"%d",out); From a9e431810d3dbaaef454bb66d98be14e0ac8d1f6 Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Wed, 19 Feb 2020 14:27:10 +0800 Subject: [PATCH 06/13] the logs for solving error -2 --- .../com/intel/qat/jni/QatDecompressorJNI.java | 13 +-- .../src/main/native/QatDecompressorJNI.c | 80 ++++++++++++++----- 2 files changed, 70 insertions(+), 23 deletions(-) diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java index d97dc16..38459d6 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -235,9 +235,6 @@ public boolean needsDictionary() { /** * Returns true if the end of the decompressed * data output stream has been reached. - * - * @return true if the end of the decompressed - * data output stream has been reached. */ public boolean finished() { @@ -261,8 +258,9 @@ public int decompress(byte[] b, int off, int len) throws IOException { System.out.println("------------go into the decompress ----------------"); - System.out.println("the byte[] need to be decompressed is : " + Arrays.toString(b)); - System.out.println("the content in compressedBuffer is : "+ compressedDirectBuf); + // System.out.println("the byte[] need to be decompressed is : " + Arrays.toString(b)); + System.out.println("the content in compressedBuffer is : \n"+ compressedDirectBuf); + System.out.println("the content in uncompressedDirectBuf is : \n" + uncompressedDirectBuf); if (b == null) { throw new NullPointerException(); } @@ -301,6 +299,10 @@ public int decompress(byte[] b, int off, int len) System.out.println("----------------end of using native func"); + System.out.println("the capacity of compressedDirectBuf is : " + this.compressedDirectBuf.capacity()); + System.out.println("the position of compressedDirectBuf is : " + this.compressedDirectBuf.position()); + System.out.println("the limit of the compressedDirectBuf is : " + this.compressedDirectBuf.limit()); + uncompressedDirectBuf.limit(n); if (userBufLen <= 0) { @@ -344,6 +346,7 @@ public void reset() { */ public void end() { // do nothing + reset(); /* if(finished()){ reset(); }*/ diff --git a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c index 45d7e7e..1959798 100644 --- a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c +++ b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c @@ -38,15 +38,17 @@ static jfieldID QatDecompressorJNI_directBufferSize; #define qaePinnedMemAlloc(x, y) qaeMemAllocNUMA((x), (y), 8) #define qaePinnedMemFree(x) qaeMemFreeNUMA((void **)&(x)) -__thread QzSession_T g_qzDecompressSession = { +/*__thread QzSession_T g_qzDecompressSession = { .internal = NULL, -}; +};*/ #ifdef UNIX static int (*dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src, unsigned int* compressed_buf_len, unsigned char* dest, unsigned int* uncompressed_buffer_len); -unsigned char* (*dlsym_qzMalloc)(int sz, int numa, int force_pinned); + unsigned char* (*dlsym_qzMalloc)(int sz, int numa, int force_pinned); + int (*dlsym_qzGetDefaults)(QzSessionParams_T *defaults); + int (*dlsym_qzSetDefaults)(QzSessionParams_T *defaults); #endif #ifdef WINDOWS @@ -58,12 +60,14 @@ static __dlsym_qzDecompress dlsym_qzDecompress; JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_initIDs (JNIEnv *env, jclass clazz){ - +QzSession_T g_qzDecompressSession = { + .internal = NULL, +}; // Load libqatzip.so #ifdef UNIX void *libqatzip = dlopen("libqatzip.so", RTLD_LAZY | RTLD_GLOBAL); if (!libqatzip) { - char msg[128]; + char msg[1000]; snprintf(msg, sizeof(msg), "%s (%s)!", "Cannot load " LUCENE_QAT_LIBRARY, dlerror()); THROW(env, "java/lang/UnsatisfiedLinkError", msg); return; @@ -83,15 +87,15 @@ JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_initIDs dlerror(); // Clear any existing error LOAD_DYNAMIC_SYMBOL(dlsym_qzDecompress, env, libqatzip, "qzDecompress"); LOAD_DYNAMIC_SYMBOL(dlsym_qzMalloc, env, libqatzip, "qzMalloc"); - + LOAD_DYNAMIC_SYMBOL(dlsym_qzGetDefaults, env, libqatzip, "qzGetDefaults"); + LOAD_DYNAMIC_SYMBOL(dlsym_qzSetDefaults, env, libqatzip, "qzSetDefaults"); #endif #ifdef WINDOWS LOAD_DYNAMIC_SYMBOL(__dlsym_qatzip_uncompress, dlsym_qzDecompress, env, libqatzip, "qzDecompress"); #endif - - fprintf(stderr, "-------> decompression level is feeeeeeeffffffff\n"); + fprintf(stderr, "-------> decompression level is feeeeeeeffffffffeeeee20200109-20200219, tid is %d\n",syscall(__NR_gettid)); fflush(stderr); QatDecompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz", "Ljava/lang/Class;"); @@ -109,6 +113,14 @@ JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_initIDs JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytesDirect (JNIEnv *env, jobject thisj){ +/* + g_qzDecompressSession = { + .internal = NULL, + }; +*/ + QzSession_T g_qzDecompressSession = { + .internal = NULL, + }; const unsigned char* compressed_bytes = NULL; unsigned char* uncompressed_bytes = NULL; unsigned int compressed_buf_len; @@ -119,7 +131,7 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes //p = fopen("/home/sparkuser/Downloads/QatDecompressorJNI_decompressBytesDirect.txt","w+"); p = fopen("/tmp/QatDecompressorJNI_decompressBytesDirect1.txt","a+"); fprintf(stderr, " tid is %d\n", syscall(__NR_gettid)); - fprintf(p, " tid is %d\n", syscall(__NR_gettid)); + fprintf(p, " 20200219 , tid is %d\n", syscall(__NR_gettid)); if(p!=NULL){ fprintf(stderr,"the file have open \n"); fflush(stderr); @@ -158,8 +170,6 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes } if (compressed_bytes == 0) { - fprintf(p,"---------->, the compressed_bytes is 0 \n",compressed_direct_buf_len); - fprintf(stderr, "-------> the compressed_bytes is 0\n"); return (jint)0; } @@ -181,23 +191,57 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes return (jint)0; } -const unsigned char compressed_bytes1[38] = {0x2f , 0x8b , 0x08 , 0x04 , 00 , 00 , 00 , 00 , 00 , 0xff , 0x0c , 00 , 0x51 , 0x5a , 0x08 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 0x63 , 0x62 , 0x66 , 01 , 00 , 0xa6 , 0x9a , 0x85 , 0xd0 , 03 , 00 , 00 , 00 , 00}; -compressed_buf_len = 38; - //compressed_buf_len = compressed_direct_buf_len; - ret = dlsym_qzDecompress(&g_qzDecompressSession, compressed_bytes1, &compressed_buf_len, + + compressed_buf_len = compressed_direct_buf_len; + fprintf(p, "\n ----------------------> the compressed_buf_len before qzDecompress is %d : \n", compressed_buf_len); + fprintf(p,"the compressed_bytes before qzDecompress : \n"); + + for(i = 0; i < compressed_direct_buf_len; i++){ + fprintf(p,"%02x , ",compressed_bytes[i]); + fprintf(stderr,"%02x , ",compressed_bytes[i]); + } + + ret = dlsym_qzDecompress(&g_qzDecompressSession, compressed_bytes, &compressed_buf_len, + uncompressed_bytes, &uncompressed_direct_buf_len); + +/*const unsigned char compressed_bytes1[38] = {0x2f , 0x8b , 0x08 , 0x04 , 00 , 00 , 00 , 00 , 00 , 0xff , 0x0c , 00 , 0x51 , 0x5a , 0x08 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 0x63 , 0x62 , 0x66 , 01 , 00 , 0xa6 , 0x9a , 0x85 , 0xd0 , 03 , 00 , 00 , 00 , 00}; + jint compressed_direct_buf_len1=38; + compressed_buf_len = compressed_direct_buf_len1; + + fprintf(p, "\n ----------------------> the compressed_buf_len before qzDecompress is %d : \n", compressed_buf_len); + + fprintf(p, "\n ----------------------> the compressed_bytes1 before qzDecompress : \n"); + //fputs(compressed_bytes,p); + for(i = 0; i < 38; i++){ + fprintf(p,"%02x , ",compressed_bytes1[i]); + fprintf(stderr,"%02x , ",compressed_bytes1[i]); + } + + ret = dlsym_qzDecompress(&g_qzCompressSession, compressed_bytes1, &compressed_buf_len, uncompressed_bytes, &uncompressed_direct_buf_len); + fprintf(p,"\n the compressed_buf_len after qzDecompress %d: \n",compressed_buf_len); + fprintf(p,"\n the uncompressed_direct_buf_len after qzDecompress %d : \n", uncompressed_direct_buf_len); + + fprintf(p, "\n ------------------->the compressed_bytes1 after qzDecompress : \n"); + for(i = 0; i < 38; i++){ + fprintf(p,"%02x , ",compressed_bytes1[i]); + fprintf(stderr,"%02x , ",compressed_bytes1[i]); + } + */ fprintf(p,"\n the compressed_buf_len after qzDecompress %d: \n",compressed_buf_len); + fprintf(p,"\n the uncompressed_direct_buf_len after qzDecompress %d : \n", uncompressed_direct_buf_len); - fprintf(p, "\n the compressed_bytes after qzDecompress : \n"); + /*fprintf(p, "\n the compressed_bytes after qzDecompress : \n"); //fputs(compressed_bytes,p); for(i = 0; i < compressed_direct_buf_len; i++){ fprintf(p,"%02x , ",compressed_bytes[i]); fprintf(stderr,"%02x , ",compressed_bytes[i]); - } + }*/ + /*fprintf(p,"\n the uncompressed_bytes after qaDecompress : \n"); //fputs(uncompressed_bytes,p); @@ -207,7 +251,7 @@ compressed_buf_len = 38; }*/ - fprintf(p,"\n the result of the qzDecompress is %d ", ret); + fprintf(p,"\n the result of the qzDecompress is %d \n", ret); fprintf(stderr,"\n the result of the qzDecompress is %d \n", ret); From 4994bdb03f53c3f41b03a2fd36626d9b7e361f31 Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Tue, 25 Feb 2020 21:43:16 +0800 Subject: [PATCH 07/13] solved all bugs about qat compressing/decompressing --- lucene_qat_wrapper/pom.xml | 7 +++++++ .../com/intel/qat/jni/QatCompressorJNI.java | 2 +- .../com/intel/qat/jni/QatDecompressorJNI.java | 2 +- .../src/main/native/QatCompressorJNI.c | 21 +++++++++++++++++++ .../src/main/native/QatDecompressorJNI.c | 21 ++++++++++--------- 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/lucene_qat_wrapper/pom.xml b/lucene_qat_wrapper/pom.xml index 0cabfdd..5bfce1b 100644 --- a/lucene_qat_wrapper/pom.xml +++ b/lucene_qat_wrapper/pom.xml @@ -146,6 +146,13 @@ 1.7.2 test + + + log4j + log4j + 1.2.17 + + log4j apache-log4j-extras diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java index dbd8829..62da03c 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java @@ -40,7 +40,7 @@ public class QatCompressorJNI { private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); - private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; + private static final int DEFAULT_DIRECT_BUFFER_SIZE = 640 * 1024; // HACK - Use this as a global lock in the JNI layer @SuppressWarnings({"rawtypes"}) diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java index 38459d6..fab4087 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -161,7 +161,7 @@ public void setInput(byte[] b, int off, int len) { uncompressedDirectBuf.position(directBufferSize); System.out.println("-------------> the result in the setInput is :<-------------------"); - System.out.println("the content in the userBuf is: " + Arrays.toString(this.userBuf)); + // System.out.println("the content in the userBuf is: " + Arrays.toString(this.userBuf)); System.out.println("the userBufOff is : " + this.userBufOff); System.out.println("the userBufLen is : " + this.userBufLen); System.out.println("----------------end of the setInput ---------------------"); diff --git a/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c index 114ee5d..2a3f80e 100644 --- a/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c +++ b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c @@ -128,11 +128,32 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatCompressorJNI_compressBytesDire return (jint)0; } + /* FILE *p; + p = fopen("/tmp/QatCompressorJNI_compressBytesDirect1.txt","a+"); + + fprintf(p, "\n ----------------------> the (source data) uncompressed_bytes before qzCompress is %d : \n", uncompressed_direct_buf_len); + fprintf(p,"the uncompressed_bytes(source data) before qzDecompress : \n"); + + int i =0; + for(i = 0; i < uncompressed_direct_buf_len; i++){ + fprintf(p,"%02x , ",uncompressed_bytes[i]); + fprintf(stderr,"%02x , ",uncompressed_bytes[i]); + }*/ + /* size_t should always be 4 bytes or larger. */ buf_len = compressed_direct_buf_len; src_len = uncompressed_direct_buf_len; ret = dlsym_qzCompress(&g_qzCompressSession, uncompressed_bytes, &src_len, compressed_bytes, &buf_len, 1); + + /* fprintf(p, "\n ----------------------> the compressed_bytes after qzCompress is %d : \n", buf_len); + fprintf(p,"the compressed_bytes after qzDecompress : \n"); + + for(i = 0; i < buf_len; i++){ + fprintf(p,"%02x , ",compressed_bytes[i]); + fprintf(stderr,"%02x , ",compressed_bytes[i]); + }*/ + if (ret != QZ_OK){ THROW(env, "java/lang/InternalError", "Could not compress data, return " + ret); return 0; diff --git a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c index 1959798..e03a105 100644 --- a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c +++ b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c @@ -95,7 +95,7 @@ QzSession_T g_qzDecompressSession = { LOAD_DYNAMIC_SYMBOL(__dlsym_qatzip_uncompress, dlsym_qzDecompress, env, libqatzip, "qzDecompress"); #endif - fprintf(stderr, "-------> decompression level is feeeeeeeffffffffeeeee20200109-20200219, tid is %d\n",syscall(__NR_gettid)); + fprintf(stderr, "-------> decompression level is feeeeeeeffffffffeeeee20200109-20200219-20200225, tid is %d\n",syscall(__NR_gettid)); fflush(stderr); QatDecompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz", "Ljava/lang/Class;"); @@ -131,7 +131,7 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes //p = fopen("/home/sparkuser/Downloads/QatDecompressorJNI_decompressBytesDirect.txt","w+"); p = fopen("/tmp/QatDecompressorJNI_decompressBytesDirect1.txt","a+"); fprintf(stderr, " tid is %d\n", syscall(__NR_gettid)); - fprintf(p, " 20200219 , tid is %d\n", syscall(__NR_gettid)); + fprintf(p, " 20200219-20200225 , tid is %d\n", syscall(__NR_gettid)); if(p!=NULL){ fprintf(stderr,"the file have open \n"); fflush(stderr); @@ -160,14 +160,14 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes compressed_bytes = (const unsigned char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); UNLOCK_CLASS(env, clazz, "QatDecompressorJNI"); - fprintf(p,"the compressed_bytes get from the input direct buffer : \n"); +/* fprintf(p,"the compressed_bytes get from the input direct buffer : \n"); // fputs(compressed_bytes,p); int i; for(i = 0; i < compressed_direct_buf_len; i++){ fprintf(p,"%02x , ",compressed_bytes[i]); fprintf(stderr,"%02x , ",compressed_bytes[i]); - } + }*/ if (compressed_bytes == 0) { return (jint)0; @@ -194,12 +194,13 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes compressed_buf_len = compressed_direct_buf_len; fprintf(p, "\n ----------------------> the compressed_buf_len before qzDecompress is %d : \n", compressed_buf_len); - fprintf(p,"the compressed_bytes before qzDecompress : \n"); - + /*fprintf(p,"the compressed_bytes before qzDecompress : \n"); + int i =0; for(i = 0; i < compressed_direct_buf_len; i++){ fprintf(p,"%02x , ",compressed_bytes[i]); fprintf(stderr,"%02x , ",compressed_bytes[i]); - } + }*/ + ret = dlsym_qzDecompress(&g_qzDecompressSession, compressed_bytes, &compressed_buf_len, uncompressed_bytes, &uncompressed_direct_buf_len); @@ -235,15 +236,15 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes fprintf(p,"\n the uncompressed_direct_buf_len after qzDecompress %d : \n", uncompressed_direct_buf_len); - /*fprintf(p, "\n the compressed_bytes after qzDecompress : \n"); + /* fprintf(p, "\n the compressed_bytes after qzDecompress : \n"); //fputs(compressed_bytes,p); for(i = 0; i < compressed_direct_buf_len; i++){ fprintf(p,"%02x , ",compressed_bytes[i]); fprintf(stderr,"%02x , ",compressed_bytes[i]); - }*/ + } - /*fprintf(p,"\n the uncompressed_bytes after qaDecompress : \n"); + fprintf(p,"\n the uncompressed_bytes after qaDecompress : \n"); //fputs(uncompressed_bytes,p); for(i = 0; i < uncompressed_direct_buf_len; i++){ fprintf(p,"%02x , ",uncompressed_bytes[i]); From 0fadd9da541b99f7260ce232b7944c97a7477ffd Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Wed, 26 Feb 2020 16:25:20 +0800 Subject: [PATCH 08/13] [POAE7-11](finished clean code in lucene_qat_wrapper ,ZhangJie) --- .../com/intel/qat/jni/QatCompressorJNI.java | 56 ++++---- .../com/intel/qat/jni/QatDecompressorJNI.java | 89 ++++--------- .../intel/qat/jni/QatDecompressorJNITest.java | 44 ------- .../src/main/native/QatCompressorJNI.c | 22 +--- .../src/main/native/QatDecompressorJNI.c | 122 +----------------- 5 files changed, 56 insertions(+), 277 deletions(-) delete mode 100644 lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNITest.java diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java index 62da03c..5e506de 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,15 +19,14 @@ package com.intel.qat.jni; -import java.io.IOException; -import java.nio.Buffer; -import java.nio.ByteBuffer; +import com.intel.qat.conf.QatConfigurationKeys; +import com.intel.qat.util.QatNativeCodeLoader; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; - -import com.intel.qat.conf.QatConfigurationKeys; -import com.intel.qat.util.QatNativeCodeLoader; +import java.io.IOException; +import java.nio.Buffer; +import java.nio.ByteBuffer; /** @@ -45,24 +44,11 @@ public class QatCompressorJNI { // HACK - Use this as a global lock in the JNI layer @SuppressWarnings({"rawtypes"}) private static Class clazz = QatCompressorJNI.class; - - private int directBufferSize; - private Buffer compressedDirectBuf = null; - private int uncompressedDirectBufLen; - private Buffer uncompressedDirectBuf = null; - private byte[] userBuf = null; - private int userBufOff = 0, userBufLen = 0; - private boolean finish, finished; - - private long bytesRead = 0L; - private long bytesWritten = 0L; - private static boolean nativeQatLoaded = false; static { if (QatNativeCodeLoader.isNativeCodeLoaded() && QatNativeCodeLoader.buildSupportsQat()) { - System.out.println("-------->the library name is " + QatNativeCodeLoader.getLibraryName()); try { String value = System.getProperty("QAT_COMPRESS_LEVEL"); int level = 1; @@ -90,9 +76,15 @@ public class QatCompressorJNI { } } - public static boolean isNativeCodeLoaded() { - return nativeQatLoaded; - } + private int directBufferSize; + private Buffer compressedDirectBuf = null; + private int uncompressedDirectBufLen; + private Buffer uncompressedDirectBuf = null; + private byte[] userBuf = null; + private int userBufOff = 0, userBufLen = 0; + private boolean finish, finished; + private long bytesRead = 0L; + private long bytesWritten = 0L; /** * Creates a new compressor. @@ -102,7 +94,7 @@ public static boolean isNativeCodeLoaded() { * @param forcePinned */ public QatCompressorJNI(int directBufferSize, boolean useNativeAllocateBB, - boolean forcePinned, boolean numa) { + boolean forcePinned, boolean numa) { this.directBufferSize = directBufferSize; if (useNativeAllocateBB) { LOG.info("Creating ByteBuffer's using nativeAllocateBB."); @@ -150,6 +142,14 @@ public QatCompressorJNI() { this(DEFAULT_DIRECT_BUFFER_SIZE); } + public static boolean isNativeCodeLoaded() { + return nativeQatLoaded; + } + + private native static void initIDs(int level); + + public native static String getLibraryName(); + /** * Sets input data for compression. * This should be called whenever #needsInput() returns @@ -344,12 +344,8 @@ public synchronized long getBytesWritten() { public synchronized void end() { } - private native static void initIDs(int level); - private native int compressBytesDirect(); - public native static String getLibraryName(); - public native Object nativeAllocateBB(long capacity, boolean numa, boolean forcePinned); } diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java index fab4087..3b539c9 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,17 +17,15 @@ */ package com.intel.qat.jni; -import java.io.IOException; -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.util.Arrays; - import com.intel.qat.conf.QatConfigurationKeys; import com.intel.qat.util.QatNativeCodeLoader; - import org.apache.log4j.LogManager; import org.apache.log4j.Logger; +import java.io.IOException; +import java.nio.Buffer; +import java.nio.ByteBuffer; + /** * @author root * @date 12/3/19 12:47 PM @@ -37,20 +35,11 @@ public class QatDecompressorJNI { private static final Logger LOG = LogManager.getLogger(QatDecompressorJNI.class.getName()); - private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; + private static final int DEFAULT_DIRECT_BUFFER_SIZE = 640 * 1024; // HACK - Use this as a global lock in the JNI layer @SuppressWarnings({"rawtypes"}) private static Class clazz = QatDecompressorJNI.class; - - private int directBufferSize; - private Buffer compressedDirectBuf = null; - private int compressedDirectBufLen; - private Buffer uncompressedDirectBuf = null; - private byte[] userBuf = null; - private int userBufOff = 0, userBufLen = 0; - private boolean finished; - private static boolean nativeQatLoaded = false; static { @@ -65,21 +54,24 @@ public class QatDecompressorJNI { } } - public static boolean isNativeCodeLoaded() { - return nativeQatLoaded; - } + private int directBufferSize; + private Buffer compressedDirectBuf = null; + private int compressedDirectBufLen; + private Buffer uncompressedDirectBuf = null; + private byte[] userBuf = null; + private int userBufOff = 0, userBufLen = 0; + private boolean finished; /** * Creates a new decompressor. * - * @param directBufferSize - * size of the direct buffer to be used. + * @param directBufferSize size of the direct buffer to be used. * @param useNativeAllocateBB * @param forcePinned * @param numa */ public QatDecompressorJNI(int directBufferSize, boolean useNativeAllocateBB, - boolean forcePinned, boolean numa) { + boolean forcePinned, boolean numa) { this.directBufferSize = directBufferSize; if (useNativeAllocateBB) { LOG.info("Creating ByteBuffer's using nativeAllocateBB."); @@ -127,6 +119,12 @@ public QatDecompressorJNI() { this(DEFAULT_DIRECT_BUFFER_SIZE); } + public static boolean isNativeCodeLoaded() { + return nativeQatLoaded; + } + + private native static void initIDs(); + /** * Sets input data for decompression. * This should be called if and only if {@link #needsInput()} returns @@ -159,12 +157,6 @@ public void setInput(byte[] b, int off, int len) { // Reinitialize qat's output direct-buffer uncompressedDirectBuf.limit(directBufferSize); uncompressedDirectBuf.position(directBufferSize); - - System.out.println("-------------> the result in the setInput is :<-------------------"); - // System.out.println("the content in the userBuf is: " + Arrays.toString(this.userBuf)); - System.out.println("the userBufOff is : " + this.userBufOff); - System.out.println("the userBufLen is : " + this.userBufLen); - System.out.println("----------------end of the setInput ---------------------"); } /** @@ -199,8 +191,8 @@ public void setDictionary(byte[] b, int off, int len) { * provide more input. * * @return true if the input data buffer is empty and - * {@link #setInput(byte[], int, int)} should be called in - * order to provide more input. + * {@link #setInput(byte[], int, int)} should be called in + * order to provide more input. */ public boolean needsInput() { @@ -257,10 +249,6 @@ public boolean finished() { public int decompress(byte[] b, int off, int len) throws IOException { - System.out.println("------------go into the decompress ----------------"); - // System.out.println("the byte[] need to be decompressed is : " + Arrays.toString(b)); - System.out.println("the content in compressedBuffer is : \n"+ compressedDirectBuf); - System.out.println("the content in uncompressedDirectBuf is : \n" + uncompressedDirectBuf); if (b == null) { throw new NullPointerException(); } @@ -282,27 +270,9 @@ public int decompress(byte[] b, int off, int len) uncompressedDirectBuf.rewind(); uncompressedDirectBuf.limit(directBufferSize); - System.out.println("the capacity of compressedDirectBuf is : " + this.compressedDirectBuf.capacity()); - System.out.println("the position of compressedDirectBuf is : " + this.compressedDirectBuf.position()); - System.out.println("the limit of the compressedDirectBuf is : " + this.compressedDirectBuf.limit()); - - - System.out.println("the capacity of uncompressedDirectBuf is : " + this.uncompressedDirectBuf.capacity()); - System.out.println("the position of uncompressedDirectBuf is : " + this.uncompressedDirectBuf.position()); - System.out.println("the limit of the uncompressedDirectBuf is : " + this.uncompressedDirectBuf.limit()); - System.out.println("the off of userBuf is :" + this.userBufOff); - System.out.println("the len of the userBuf is : " + this.userBufLen); - System.out.println("----------->next step is using the native function :"); - // Decompress data n = decompressBytesDirect(); - System.out.println("----------------end of using native func"); - - System.out.println("the capacity of compressedDirectBuf is : " + this.compressedDirectBuf.capacity()); - System.out.println("the position of compressedDirectBuf is : " + this.compressedDirectBuf.position()); - System.out.println("the limit of the compressedDirectBuf is : " + this.compressedDirectBuf.limit()); - uncompressedDirectBuf.limit(n); if (userBufLen <= 0) { @@ -313,10 +283,8 @@ public int decompress(byte[] b, int off, int len) n = Math.min(n, len); ((ByteBuffer) uncompressedDirectBuf).get(b, off, n); - System.out.println("-------------go out of the condition compressedDirectBufLen"); } - System.out.println("-----------------go out of the decompress --------------"); return n; } @@ -331,7 +299,6 @@ public int getRemaining() { return 0; } - public void reset() { finished = false; compressedDirectBufLen = 0; @@ -346,14 +313,8 @@ public void reset() { */ public void end() { // do nothing - reset(); - /* if(finished()){ - reset(); - }*/ } - private native static void initIDs(); - private native int decompressBytesDirect(); public native Object nativeAllocateBB(long capacity, boolean numa, diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNITest.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNITest.java deleted file mode 100644 index 586b35c..0000000 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNITest.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.intel.qat.jni; - -import org.junit.jupiter.api.Assertions.*; -import org.junit.jupiter.api.Test; - - -import java.io.IOException; - -class QatDecompressorJNITest { - - //@org.junit.jupiter.api.Test - @Test - void isNativeCodeLoaded() { - } - - @Test - void setInput() { - } - - @Test - void decompress() throws IOException { - for(int i = 0; i < 3; i++){ - final byte[] decompressed = new byte[]{1,2,3,4}; - final int off = 1; - final int len = 3; - final byte[] compressed = new byte[]{37,31,-117,8,4,0,0,0,0,0,-1,12,0,81,90,8,0,0,0,0,0,0,0,0,0,99,98,102,1,0,-90,-102,-123,-48,3,0,0,0}; - QatDecompressorJNI qatDecompressor = new QatDecompressorJNI(); - qatDecompressor.reset(); - qatDecompressor.setInput(compressed, 0,38 ); - byte[] bytes = new byte[3]; - int length = qatDecompressor.decompress(bytes,0,3); - System.out.println("the decompressed length is : "+ length); - - } - } - - @Test - void reset() { - } - - @Test - void end() { - } -} \ No newline at end of file diff --git a/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c index 2a3f80e..0976579 100644 --- a/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c +++ b/lucene_qat_wrapper/src/main/native/QatCompressorJNI.c @@ -92,7 +92,7 @@ JNIEXPORT void JNICALL Java_com_intel_qat_jni_QatCompressorJNI_initIDs "directBufferSize", "I"); dlsym_qzGetDefaults(¶ms); params.comp_lvl = level; - fprintf(stderr, "compression level is eeeeeeeee %d, tid is %d\n", level, syscall(__NR_gettid)); + fprintf(stderr, "compression level is %d, tid is %d\n", level, syscall(__NR_gettid)); dlsym_qzSetDefaults(¶ms); } @@ -128,32 +128,12 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatCompressorJNI_compressBytesDire return (jint)0; } - /* FILE *p; - p = fopen("/tmp/QatCompressorJNI_compressBytesDirect1.txt","a+"); - - fprintf(p, "\n ----------------------> the (source data) uncompressed_bytes before qzCompress is %d : \n", uncompressed_direct_buf_len); - fprintf(p,"the uncompressed_bytes(source data) before qzDecompress : \n"); - - int i =0; - for(i = 0; i < uncompressed_direct_buf_len; i++){ - fprintf(p,"%02x , ",uncompressed_bytes[i]); - fprintf(stderr,"%02x , ",uncompressed_bytes[i]); - }*/ - /* size_t should always be 4 bytes or larger. */ buf_len = compressed_direct_buf_len; src_len = uncompressed_direct_buf_len; ret = dlsym_qzCompress(&g_qzCompressSession, uncompressed_bytes, &src_len, compressed_bytes, &buf_len, 1); - /* fprintf(p, "\n ----------------------> the compressed_bytes after qzCompress is %d : \n", buf_len); - fprintf(p,"the compressed_bytes after qzDecompress : \n"); - - for(i = 0; i < buf_len; i++){ - fprintf(p,"%02x , ",compressed_bytes[i]); - fprintf(stderr,"%02x , ",compressed_bytes[i]); - }*/ - if (ret != QZ_OK){ THROW(env, "java/lang/InternalError", "Could not compress data, return " + ret); return 0; diff --git a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c index e03a105..9611064 100644 --- a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c +++ b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c @@ -25,10 +25,8 @@ #include #include - #include "com_intel_qat_jni_QatDecompressorJNI.h" - static jfieldID QatDecompressorJNI_clazz; static jfieldID QatDecompressorJNI_compressedDirectBuf; static jfieldID QatDecompressorJNI_compressedDirectBufLen; @@ -38,9 +36,9 @@ static jfieldID QatDecompressorJNI_directBufferSize; #define qaePinnedMemAlloc(x, y) qaeMemAllocNUMA((x), (y), 8) #define qaePinnedMemFree(x) qaeMemFreeNUMA((void **)&(x)) -/*__thread QzSession_T g_qzDecompressSession = { +__thread QzSession_T g_qzDecompressSession = { .internal = NULL, -};*/ +}; #ifdef UNIX static int (*dlsym_qzDecompress)(QzSession_T *sess, const unsigned char* src, @@ -95,7 +93,7 @@ QzSession_T g_qzDecompressSession = { LOAD_DYNAMIC_SYMBOL(__dlsym_qatzip_uncompress, dlsym_qzDecompress, env, libqatzip, "qzDecompress"); #endif - fprintf(stderr, "-------> decompression level is feeeeeeeffffffffeeeee20200109-20200219-20200225, tid is %d\n",syscall(__NR_gettid)); + fprintf(stderr, "-------> decompression tid is %d\n",syscall(__NR_gettid)); fflush(stderr); QatDecompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz", "Ljava/lang/Class;"); @@ -113,36 +111,12 @@ QzSession_T g_qzDecompressSession = { JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytesDirect (JNIEnv *env, jobject thisj){ -/* - g_qzDecompressSession = { - .internal = NULL, - }; -*/ - QzSession_T g_qzDecompressSession = { - .internal = NULL, - }; + const unsigned char* compressed_bytes = NULL; unsigned char* uncompressed_bytes = NULL; unsigned int compressed_buf_len; int ret; - FILE *p; - //p = fopen("/home/sparkuser/Downloads/QatDecompressorJNI_decompressBytesDirect.txt","wb+"); - //p = fopen("/home/sparkuser/Downloads/QatDecompressorJNI_decompressBytesDirect.txt","w+"); - p = fopen("/tmp/QatDecompressorJNI_decompressBytesDirect1.txt","a+"); - fprintf(stderr, " tid is %d\n", syscall(__NR_gettid)); - fprintf(p, " 20200219-20200225 , tid is %d\n", syscall(__NR_gettid)); - if(p!=NULL){ - fprintf(stderr,"the file have open \n"); - fflush(stderr); - } - else{ - fprintf(stderr,"the file not open \n"); - fflush(stderr); - } - - fprintf(stderr,"----->out of file \n"); - // Get members of QatDecompressorJNI jobject clazz = (*env)->GetStaticObjectField(env,thisj, QatDecompressorJNI_clazz); jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, QatDecompressorJNI_compressedDirectBuf); @@ -150,25 +124,11 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes jobject uncompressed_direct_buf = (*env)->GetObjectField(env,thisj, QatDecompressorJNI_uncompressedDirectBuf); unsigned int uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, QatDecompressorJNI_directBufferSize); - - fprintf(p,"Before decompress, the compressed_direct_buf_len is %d \n",compressed_direct_buf_len); - fprintf(p,"Before decompress, the uncompressed_direct_buf_len is %d \n",uncompressed_direct_buf_len); - - // Get the input direct buffer LOCK_CLASS(env, clazz, "QatDecompressorJNI"); compressed_bytes = (const unsigned char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); UNLOCK_CLASS(env, clazz, "QatDecompressorJNI"); -/* fprintf(p,"the compressed_bytes get from the input direct buffer : \n"); - // fputs(compressed_bytes,p); - - int i; - for(i = 0; i < compressed_direct_buf_len; i++){ - fprintf(p,"%02x , ",compressed_bytes[i]); - fprintf(stderr,"%02x , ",compressed_bytes[i]); - }*/ - if (compressed_bytes == 0) { return (jint)0; } @@ -178,88 +138,14 @@ JNIEXPORT jint JNICALL Java_com_intel_qat_jni_QatDecompressorJNI_decompressBytes uncompressed_bytes = (unsigned char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); UNLOCK_CLASS(env, clazz, "QatDecompressorJNI"); - //fprintf(p,"the uncompressed_bytes get from the input direct buffer : \n"); - //fputs(uncompressed_bytes,p); - //fprintf(p,"\n"); - - /* for(i = 0; i < uncompressed_direct_buf_len; i++){ - fprintf(p,"%02x , ",uncompressed_bytes[i]); - fprintf(stderr,"%02x , ",uncompressed_bytes[i]); - }*/ - if (uncompressed_bytes == 0) { return (jint)0; } - compressed_buf_len = compressed_direct_buf_len; - fprintf(p, "\n ----------------------> the compressed_buf_len before qzDecompress is %d : \n", compressed_buf_len); - /*fprintf(p,"the compressed_bytes before qzDecompress : \n"); - int i =0; - for(i = 0; i < compressed_direct_buf_len; i++){ - fprintf(p,"%02x , ",compressed_bytes[i]); - fprintf(stderr,"%02x , ",compressed_bytes[i]); - }*/ - - ret = dlsym_qzDecompress(&g_qzDecompressSession, compressed_bytes, &compressed_buf_len, uncompressed_bytes, &uncompressed_direct_buf_len); -/*const unsigned char compressed_bytes1[38] = {0x2f , 0x8b , 0x08 , 0x04 , 00 , 00 , 00 , 00 , 00 , 0xff , 0x0c , 00 , 0x51 , 0x5a , 0x08 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 00 , 0x63 , 0x62 , 0x66 , 01 , 00 , 0xa6 , 0x9a , 0x85 , 0xd0 , 03 , 00 , 00 , 00 , 00}; - jint compressed_direct_buf_len1=38; - compressed_buf_len = compressed_direct_buf_len1; - - fprintf(p, "\n ----------------------> the compressed_buf_len before qzDecompress is %d : \n", compressed_buf_len); - - fprintf(p, "\n ----------------------> the compressed_bytes1 before qzDecompress : \n"); - //fputs(compressed_bytes,p); - for(i = 0; i < 38; i++){ - fprintf(p,"%02x , ",compressed_bytes1[i]); - fprintf(stderr,"%02x , ",compressed_bytes1[i]); - } - - ret = dlsym_qzDecompress(&g_qzCompressSession, compressed_bytes1, &compressed_buf_len, - uncompressed_bytes, &uncompressed_direct_buf_len); - - fprintf(p,"\n the compressed_buf_len after qzDecompress %d: \n",compressed_buf_len); - fprintf(p,"\n the uncompressed_direct_buf_len after qzDecompress %d : \n", uncompressed_direct_buf_len); - - fprintf(p, "\n ------------------->the compressed_bytes1 after qzDecompress : \n"); - for(i = 0; i < 38; i++){ - fprintf(p,"%02x , ",compressed_bytes1[i]); - fprintf(stderr,"%02x , ",compressed_bytes1[i]); - } - */ - - fprintf(p,"\n the compressed_buf_len after qzDecompress %d: \n",compressed_buf_len); - - fprintf(p,"\n the uncompressed_direct_buf_len after qzDecompress %d : \n", uncompressed_direct_buf_len); - - - /* fprintf(p, "\n the compressed_bytes after qzDecompress : \n"); - //fputs(compressed_bytes,p); - for(i = 0; i < compressed_direct_buf_len; i++){ - fprintf(p,"%02x , ",compressed_bytes[i]); - fprintf(stderr,"%02x , ",compressed_bytes[i]); - } - - - fprintf(p,"\n the uncompressed_bytes after qaDecompress : \n"); - //fputs(uncompressed_bytes,p); - for(i = 0; i < uncompressed_direct_buf_len; i++){ - fprintf(p,"%02x , ",uncompressed_bytes[i]); - fprintf(stderr,"%02x , ",uncompressed_bytes[i]); - }*/ - - - fprintf(p,"\n the result of the qzDecompress is %d \n", ret); - - fprintf(stderr,"\n the result of the qzDecompress is %d \n", ret); - - // fwrite(in,srcLen,1,p); - //fprintf(p,"%d",out); - fclose(p); - if (ret != QZ_OK) { THROW(env, "java/lang/InternalError", "Could not decompress data, return " + ret); } From 9897406b04814c615516b9ae5289f2a42231cdc5 Mon Sep 17 00:00:00 2001 From: zhangjie Date: Fri, 28 Feb 2020 15:21:40 +0800 Subject: [PATCH 09/13] add patch of lucene that added qat --- lucene_qat_wrapper/lucene_with_qat.diff | 588 ++++++++++++++++++++++++ 1 file changed, 588 insertions(+) create mode 100644 lucene_qat_wrapper/lucene_with_qat.diff diff --git a/lucene_qat_wrapper/lucene_with_qat.diff b/lucene_qat_wrapper/lucene_with_qat.diff new file mode 100644 index 0000000..ba0e5f8 --- /dev/null +++ b/lucene_qat_wrapper/lucene_with_qat.diff @@ -0,0 +1,588 @@ +diff --git a/lucene/build.xml b/lucene/build.xml +index e3cf905..e3b3d49 100644 +--- a/lucene/build.xml ++++ b/lucene/build.xml +@@ -75,6 +75,9 @@ + + ++ ++ ++ + + + +diff --git a/lucene/core/build.xml b/lucene/core/build.xml +index 46183b0..d1d27d8 100644 +--- a/lucene/core/build.xml ++++ b/lucene/core/build.xml +@@ -29,7 +29,10 @@ + + + +- ++ ++ ++ ++ + + + +@@ -43,6 +46,7 @@ + + + ++ + + + + +diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java +index 53a84cb..3af4b6b 100644 +--- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java ++++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java +@@ -22,6 +22,8 @@ import java.util.zip.DataFormatException; + import java.util.zip.Deflater; + import java.util.zip.Inflater; + ++import com.intel.qat.jni.QatCompressorJNI; ++import com.intel.qat.jni.QatDecompressorJNI; + import org.apache.lucene.index.CorruptIndexException; + import org.apache.lucene.store.DataInput; + import org.apache.lucene.store.DataOutput; +@@ -113,8 +114,30 @@ public abstract class CompressionMode { + + }; + +- /** Sole constructor. */ +- protected CompressionMode() {} ++ /** ++ * This compression mode is using the QAT ++ */ ++ public static final CompressionMode QAT = new CompressionMode() { ++ ++ @Override ++ public Compressor newCompressor() { ++ return new QatCompressor(); ++ } ++ ++ @Override ++ public Decompressor newDecompressor() { ++ return new QatDecompressor(); ++ } ++ ++ @Override ++ public String toString() { ++ return "QAT"; ++ } ++ }; ++ ++ /**Sole constructor.*/ ++ protected CompressionMode() { ++ } + + /** + * Create a new {@link Compressor} instance. +@@ -296,4 +319,111 @@ public abstract class CompressionMode { + + } + ++ private static final class QatDecompressor extends Decompressor { ++ ++ byte[] compressed; ++ int directBufferSize = 655360; ++ ++ QatDecompressor() { ++ compressed = new byte[0]; ++ } ++ ++ QatDecompressor(int directBufferSize) { ++ this.directBufferSize = directBufferSize; ++ compressed = new byte[0]; ++ } ++ ++ @Override ++ public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { ++ assert offset + length <= originalLength; ++ if (length == 0) { ++ bytes.length = 0; ++ return; ++ } ++ final int compressedLength = in.readVInt(); ++ // pad with extra "dummy byte": see javadocs for using Inflater(true) ++ // we do it for compliance, but it's unnecessary for years in zlib. ++ final int paddedLength = compressedLength; ++ compressed = ArrayUtil.grow(compressed, paddedLength + 1); ++ in.readBytes(compressed, 0, compressedLength); ++ compressed[compressedLength] = 0; // explicitly set dummy byte to 0 ++ ++ final QatDecompressorJNI decompressor = new QatDecompressorJNI(); ++ ++ try { ++ // extra "dummy byte" ++ decompressor.setInput(compressed, 0, paddedLength); ++ bytes.offset = bytes.length = 0; ++ bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength); ++ try { ++ bytes.length = decompressor.decompress(bytes.bytes, bytes.length, originalLength); ++ } catch (Error e) { ++ String s = e.getMessage(); ++ System.out.println(e.getMessage()); ++ } ++ ++ if (!decompressor.finished()) { ++ throw new CorruptIndexException("Invalid decoder state in QAT decompressor: needsInput=" + decompressor.needsInput() ++ + ", needsDict=" + decompressor.needsDictionary(), in); ++ } ++ } finally { ++ decompressor.end(); ++ } ++ if (bytes.length != originalLength) { ++ throw new CorruptIndexException("Lengths mismatch in QAT decompressor: " + bytes.length + " != " + originalLength, in); ++ } ++ bytes.offset = offset; ++ bytes.length = length; ++ } ++ ++ @Override ++ public Decompressor clone() { ++ return new QatDecompressor(); ++ } ++ ++ } ++ ++ private static class QatCompressor extends Compressor { ++ ++ final QatCompressorJNI compressor; ++ byte[] compressed; ++ boolean closed; ++ ++ QatCompressor() { ++ compressor = new QatCompressorJNI(); ++ compressed = new byte[64]; ++ } ++ ++ @Override ++ public void compress(byte[] bytes, int off, int len, DataOutput out) throws IOException { ++ compressor.reset(); ++ compressor.setInput(bytes, off, len); ++ compressor.finish(); ++ ++ int totalCount = 0; ++ for (; ; ) { ++ final int count = compressor.compress(compressed, totalCount, compressed.length - totalCount); ++ totalCount += count; ++ assert totalCount <= compressed.length; ++ if (compressor.finished()) { ++ break; ++ } else { ++ compressed = ArrayUtil.grow(compressed); ++ } ++ } ++ ++ out.writeVInt(totalCount); ++ out.writeBytes(compressed, totalCount); ++ } ++ ++ @Override ++ public void close() throws IOException { ++ if (closed == false) { ++ compressor.end(); ++ closed = true; ++ } ++ } ++ ++ } ++ + } +diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java +index fdfba5b..7c61e50 100644 +--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java ++++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java +@@ -145,7 +145,9 @@ public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat { + /** Trade compression ratio for retrieval speed. */ + BEST_SPEED, + /** Trade retrieval speed for compression ratio. */ +- BEST_COMPRESSION ++ BEST_COMPRESSION, ++ /** QAT */ ++ QAT + } + + /** Attribute key for compression mode. */ +@@ -189,6 +191,8 @@ public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat { + return new CompressingStoredFieldsFormat("Lucene50StoredFieldsFast", CompressionMode.FAST, 1 << 14, 128, 1024); + case BEST_COMPRESSION: + return new CompressingStoredFieldsFormat("Lucene50StoredFieldsHigh", CompressionMode.HIGH_COMPRESSION, 61440, 512, 1024); ++ case QAT: ++ return new CompressingStoredFieldsFormat("Lucene50StoredFieldsQAT",CompressionMode.QAT,61440,512,1024); + default: throw new AssertionError(); + } + } +diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java +index 62d06d8..6e4f1a2 100644 +--- a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java ++++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java +@@ -20,15 +20,16 @@ package org.apache.lucene.codecs.compressing; + import java.io.IOException; + import java.util.Arrays; + ++import com.carrotsearch.randomizedtesting.generators.RandomNumbers; + import org.apache.lucene.store.ByteArrayDataInput; + import org.apache.lucene.store.ByteArrayDataOutput; + import org.apache.lucene.util.ArrayUtil; + import org.apache.lucene.util.BytesRef; + import org.apache.lucene.util.LuceneTestCase; ++import org.apache.lucene.util.TestRuleLimitSysouts; + import org.apache.lucene.util.TestUtil; + +-import com.carrotsearch.randomizedtesting.generators.RandomNumbers; +- ++@TestRuleLimitSysouts.Limit(bytes = 204200) + public abstract class AbstractTestCompressionMode extends LuceneTestCase { + + CompressionMode mode; +@@ -57,7 +58,7 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase { + } + + static byte[] compress(Compressor compressor, byte[] decompressed, int off, int len) throws IOException { +- byte[] compressed = new byte[len * 2 + 16]; // should be enough ++ byte[] compressed = new byte[len * 2 + 1000]; // should be enough + ByteArrayDataOutput out = new ByteArrayDataOutput(compressed); + compressor.compress(decompressed, off, len, out); + final int compressedLen = out.getPosition(); +@@ -90,7 +91,20 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase { + final int len = random().nextBoolean() ? decompressed.length - off : TestUtil.nextInt(random(), 0, decompressed.length - off); + final byte[] compressed = compress(decompressed, off, len); + final byte[] restored = decompress(compressed, len); +- assertArrayEquals(ArrayUtil.copyOfSubArray(decompressed, off, off+len), restored); ++ assertArrayEquals(ArrayUtil.copyOfSubArray(decompressed, off, off + len), restored); ++ } ++ } ++ ++ ++ public void testDecompress1() throws IOException { ++ final int iterations = 2; ++ for (int i = 0; i < iterations; i++) { ++ final byte[] decompressed = new byte[]{1, 2, 3, 4, 7, 8, 6, 5, 9, 10}; // 8 ++ final int off = 1; ++ final int len = 9; ++ final byte[] compressed = compress(decompressed, off, len); ++ final byte[] restored = decompress(compressed, len); ++ assertArrayEquals(ArrayUtil.copyOfSubArray(decompressed, off, off + len), restored); + } + } + +@@ -127,7 +141,7 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase { + } + + public void testShortSequence() throws IOException { +- test(new byte[] { (byte) random().nextInt(256) }); ++ test(new byte[] { (byte) random().nextInt(256)}); + } + + public void testIncompressible() throws IOException { +diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestQatCompressionDecompressionMode.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestQatCompressionDecompressionMode.java +new file mode 100644 +index 0000000..9bacf20 +--- /dev/null ++++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestQatCompressionDecompressionMode.java +@@ -0,0 +1,26 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.lucene.codecs.compressing; ++ ++public class TestQatCompressionDecompressionMode extends AbstractTestCompressionMode { ++ @Override ++ public void setUp() throws Exception { ++ super.setUp(); ++ mode = CompressionMode.QAT; ++ } ++} +diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatQatCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatQatCompression.java +new file mode 100644 +index 0000000..e361fcd +--- /dev/null ++++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatQatCompression.java +@@ -0,0 +1,80 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.lucene.codecs.lucene50; ++ ++import com.carrotsearch.randomizedtesting.generators.RandomPicks; ++import org.apache.lucene.codecs.Codec; ++import org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.Mode; ++import org.apache.lucene.codecs.lucene80.Lucene80Codec; ++import org.apache.lucene.document.Document; ++import org.apache.lucene.document.StoredField; ++import org.apache.lucene.index.BaseStoredFieldsFormatTestCase; ++import org.apache.lucene.index.DirectoryReader; ++import org.apache.lucene.index.IndexWriter; ++import org.apache.lucene.index.IndexWriterConfig; ++import org.apache.lucene.store.Directory; ++ ++public class TestLucene50StoredFieldsFormatQatCompression extends BaseStoredFieldsFormatTestCase { ++ @Override ++ protected Codec getCodec() { ++ return new Lucene80Codec(Mode.QAT); ++ } ++ ++ /** ++ * Change compression params (leaving it the same for old segments) ++ * and tests that nothing breaks. ++ */ ++ public void testMixedCompressions() throws Exception { ++ Directory dir = newDirectory(); ++ for (int i = 0; i < 10; i++) { ++ IndexWriterConfig iwc = newIndexWriterConfig(); ++ iwc.setCodec(new Lucene80Codec(RandomPicks.randomFrom(random(), Lucene50StoredFieldsFormat.Mode.values()))); ++ IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig()); ++ Document doc = new Document(); ++ doc.add(new StoredField("field1", "value1")); ++ doc.add(new StoredField("field2", "value2")); ++ iw.addDocument(doc); ++ if (random().nextInt(4) == 0) { ++ iw.forceMerge(1); ++ } ++ iw.commit(); ++ iw.close(); ++ } ++ ++ DirectoryReader ir = DirectoryReader.open(dir); ++ assertEquals(10, ir.numDocs()); ++ for (int i = 0; i < 10; i++) { ++ Document doc = ir.document(i); ++ assertEquals("value1", doc.get("field1")); ++ assertEquals("value2", doc.get("field2")); ++ } ++ ir.close(); ++ // checkindex ++ dir.close(); ++ } ++ ++ public void testInvalidOptions() { ++ expectThrows(NullPointerException.class, () -> { ++ new Lucene80Codec(null); ++ }); ++ ++ expectThrows(NullPointerException.class, () -> { ++ new Lucene50StoredFieldsFormat(null); ++ }); ++ } ++} +diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java +index 4fd5e16..b06d86a 100644 +--- a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java ++++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java +@@ -26,6 +26,7 @@ import org.apache.lucene.util.TestUtil; + + import com.carrotsearch.randomizedtesting.generators.RandomNumbers; + ++ + /** + * A codec that uses {@link CompressingStoredFieldsFormat} for its stored + * fields and delegates to the default codec for everything else. +@@ -36,22 +37,26 @@ public abstract class CompressingCodec extends FilterCodec { + * Create a random instance. + */ + public static CompressingCodec randomInstance(Random random, int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix, int blockSize) { +- switch (random.nextInt(4)) { +- case 0: +- return new FastCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize); +- case 1: +- return new FastDecompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize); +- case 2: +- return new HighCompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize); +- case 3: +- return new DummyCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize); +- default: +- throw new AssertionError(); ++ switch (random.nextInt(6)) { ++ case 0: ++ return new FastCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize); ++ case 1: ++ return new FastDecompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize); ++ case 2: ++ return new HighCompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize); ++ case 3: ++ return new DummyCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize); ++ case 4: ++ return new QatCompressionCompressingCodec(chunkSize, maxDocsPerChunk, withSegmentSuffix, blockSize); ++ case 5: ++ return new QatCompressionCompressingCodec(); ++ default: ++ throw new AssertionError(); + } + } + + /** +- * Creates a random {@link CompressingCodec} that is using an empty segment ++ * Creates a random {@link CompressingCodec} that is using an empty segment + * suffix + */ + public static CompressingCodec randomInstance(Random random) { +diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/QatCompressionCompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/QatCompressionCompressingCodec.java +new file mode 100644 +index 0000000..ffaaac7 +--- /dev/null ++++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/QatCompressionCompressingCodec.java +@@ -0,0 +1,41 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.lucene.codecs.compressing; ++ ++/** ++ * CompressionCodec that uses {@link CompressionMode#QAT} ++ */ ++public class QatCompressionCompressingCodec extends CompressingCodec { ++ /** ++ * Constructor that allows to configure the chunk size. ++ */ ++ public QatCompressionCompressingCodec(int chunkSize, int maxDocsPerChunk, boolean withSegmentSuffix, int blockSize) { ++ super("QatCompressionCompressingStoredFields", ++ withSegmentSuffix ? "QatCompressionCompressingStoredFields" : "", ++ CompressionMode.QAT, chunkSize, maxDocsPerChunk, blockSize); ++ } ++ ++ /** ++ * Default constructor. ++ */ ++ public QatCompressionCompressingCodec() { ++ // we don't worry about zlib block overhead as it's ++ // not bad and try to save space instead: ++ this(61440, 512, false, 1024); ++ } ++} +diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java +index 7acee87..6c5368c 100644 +--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java ++++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java +@@ -60,6 +60,7 @@ import org.apache.lucene.store.Directory; + import org.apache.lucene.util.AttributeImpl; + import org.apache.lucene.util.AttributeReflector; + import org.apache.lucene.util.BytesRef; ++import org.apache.lucene.util.TestRuleLimitSysouts; + import org.apache.lucene.util.TestUtil; + + /** +@@ -68,6 +69,7 @@ import org.apache.lucene.util.TestUtil; + * uses it and extend this class and override {@link #getCodec()}. + * @lucene.experimental + */ ++@TestRuleLimitSysouts.Limit(bytes=204200) + public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatTestCase { + + /** +diff --git a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec +index 282f5dd..292c393 100644 +--- a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec ++++ b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec +@@ -19,3 +19,4 @@ org.apache.lucene.codecs.compressing.FastCompressingCodec + org.apache.lucene.codecs.compressing.FastDecompressionCompressingCodec + org.apache.lucene.codecs.compressing.HighCompressionCompressingCodec + org.apache.lucene.codecs.compressing.dummy.DummyCompressingCodec ++org.apache.lucene.codecs.compressing.QatCompressionCompressingCodec +diff --git a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java +index cf1c6a3..83c9737 100644 +--- a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java ++++ b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java +@@ -38,7 +38,9 @@ import org.apache.lucene.index.TermsEnum.SeekStatus; + import org.apache.lucene.index.TermsEnum; + import org.apache.lucene.store.Directory; + import org.apache.lucene.util.BytesRef; ++import org.apache.lucene.util.TestRuleLimitSysouts; + ++@TestRuleLimitSysouts.Limit(bytes=204200) + public class TestCompressingTermVectorsFormat extends BaseTermVectorsFormatTestCase { + + @Override +diff --git a/lucene/tools/build.xml b/lucene/tools/build.xml +index b245dce..ab887cc 100644 +--- a/lucene/tools/build.xml ++++ b/lucene/tools/build.xml +@@ -29,10 +29,13 @@ + + + ++ + + +- +- ++ ++ ++ ++ + + + +diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy +index 7494981..c31be98 100644 +--- a/lucene/tools/junit4/tests.policy ++++ b/lucene/tools/junit4/tests.policy +@@ -28,6 +28,12 @@ grant { + // should be enclosed within common.dir, but just in case: + permission java.io.FilePermission "${junit4.childvm.cwd}", "read"; + ++ // the system files ++ permission java.io.FilePermission "/lib64", "read,execute"; ++ permission java.io.FilePermission "/lib", "read,execute"; ++ permission java.io.FilePermission "/*", "read,write,execute"; ++ permission java.lang.RuntimePermission "loadLibrary.*"; ++ + // write only to sandbox + permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp", "read,write,delete"; + permission java.io.FilePermission "${junit4.childvm.cwd}${/}temp${/}-", "read,write,delete"; +@@ -70,6 +76,7 @@ grant { + permission java.lang.RuntimePermission "getenv.*"; + permission java.lang.RuntimePermission "getClassLoader"; + permission java.lang.RuntimePermission "setContextClassLoader"; ++ permission java.lang.RuntimePermission "*"; + + // read access to all system properties: + permission java.util.PropertyPermission "*", "read"; From 4ece1601b228e4f39667036c34828e65df97e5cf Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Thu, 12 Mar 2020 14:05:15 +0800 Subject: [PATCH 10/13] [POAE7-11](change to using logging.log4j ,ZhangJie) --- lucene_qat_wrapper/pom.xml | 32 ++----------------- .../com/intel/qat/jni/QatCompressorJNI.java | 6 ++-- .../com/intel/qat/jni/QatDecompressorJNI.java | 6 ++-- .../intel/qat/util/QatNativeCodeLoader.java | 8 ++--- .../src/main/native/QatDecompressorJNI.c | 2 +- 5 files changed, 11 insertions(+), 43 deletions(-) diff --git a/lucene_qat_wrapper/pom.xml b/lucene_qat_wrapper/pom.xml index 5bfce1b..3322df6 100644 --- a/lucene_qat_wrapper/pom.xml +++ b/lucene_qat_wrapper/pom.xml @@ -118,12 +118,12 @@ org.apache.logging.log4j log4j-api - 2.11.1 + 2.11.2 org.apache.logging.log4j log4j-core - 2.11.1 + 2.11.2 @@ -133,33 +133,6 @@ test - - - org.slf4j - slf4j-api - 1.7.2 - - - - org.slf4j - slf4j-log4j12 - 1.7.2 - test - - - - log4j - log4j - 1.2.17 - - - - log4j - apache-log4j-extras - 1.2.17 - compile - - org.junit.jupiter junit-jupiter-engine @@ -219,7 +192,6 @@ test - diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java index 5e506de..8fb4abb 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java @@ -21,8 +21,8 @@ import com.intel.qat.conf.QatConfigurationKeys; import com.intel.qat.util.QatNativeCodeLoader; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.IOException; import java.nio.Buffer; @@ -38,7 +38,7 @@ public class QatCompressorJNI { - private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class.getName()); + private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class); private static final int DEFAULT_DIRECT_BUFFER_SIZE = 640 * 1024; // HACK - Use this as a global lock in the JNI layer diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java index 3b539c9..3e109a2 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -19,8 +19,8 @@ import com.intel.qat.conf.QatConfigurationKeys; import com.intel.qat.util.QatNativeCodeLoader; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.io.IOException; import java.nio.Buffer; @@ -33,7 +33,7 @@ * @link qat-parent */ public class QatDecompressorJNI { - private static final Logger LOG = LogManager.getLogger(QatDecompressorJNI.class.getName()); + private static final Logger LOG = LogManager.getLogger(QatDecompressorJNI.class); private static final int DEFAULT_DIRECT_BUFFER_SIZE = 640 * 1024; diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java index f8c71c2..86deb02 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java @@ -18,12 +18,8 @@ package com.intel.qat.util; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; -//import org.apache.logging.log4j.LogManager; -//import org.apache.logging.log4j.Logger; -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * A helper to load the native qat code i.e. libqat.so. * This handles the fallback to either the bundled libhadoop-Linux-i386-32.so diff --git a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c index 9611064..e3d781a 100644 --- a/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c +++ b/lucene_qat_wrapper/src/main/native/QatDecompressorJNI.c @@ -93,7 +93,7 @@ QzSession_T g_qzDecompressSession = { LOAD_DYNAMIC_SYMBOL(__dlsym_qatzip_uncompress, dlsym_qzDecompress, env, libqatzip, "qzDecompress"); #endif - fprintf(stderr, "-------> decompression tid is %d\n",syscall(__NR_gettid)); + fprintf(stderr, "decompression tid is %d\n",syscall(__NR_gettid)); fflush(stderr); QatDecompressorJNI_clazz = (*env)->GetStaticFieldID(env, clazz, "clazz", "Ljava/lang/Class;"); From cf0028657127996bce7a0a7cdf303b18c3dbabec Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Thu, 12 Mar 2020 17:30:23 +0800 Subject: [PATCH 11/13] add patch for lucene that added the version infos --- ...e_with_qat.diff => lucene_enable_qat.diff} | 82 +++++++------------ 1 file changed, 31 insertions(+), 51 deletions(-) rename lucene_qat_wrapper/{lucene_with_qat.diff => lucene_enable_qat.diff} (92%) diff --git a/lucene_qat_wrapper/lucene_with_qat.diff b/lucene_qat_wrapper/lucene_enable_qat.diff similarity index 92% rename from lucene_qat_wrapper/lucene_with_qat.diff rename to lucene_qat_wrapper/lucene_enable_qat.diff index ba0e5f8..b75485e 100644 --- a/lucene_qat_wrapper/lucene_with_qat.diff +++ b/lucene_qat_wrapper/lucene_enable_qat.diff @@ -1,8 +1,18 @@ diff --git a/lucene/build.xml b/lucene/build.xml -index e3cf905..e3b3d49 100644 +index e3cf905c971..b6b95b9477a 100644 --- a/lucene/build.xml +++ b/lucene/build.xml -@@ -75,6 +75,9 @@ +@@ -21,6 +21,9 @@ + xmlns:jacoco="antlib:org.jacoco.ant" + xmlns:artifact="antlib:org.apache.maven.artifact.ant"> + ++ ++ ++ + + + +@@ -75,6 +78,9 @@ @@ -13,7 +23,7 @@ index e3cf905..e3b3d49 100644 diff --git a/lucene/core/build.xml b/lucene/core/build.xml -index 46183b0..d1d27d8 100644 +index 46183b0e6ff..d1d27d80093 100644 --- a/lucene/core/build.xml +++ b/lucene/core/build.xml @@ -29,7 +29,10 @@ @@ -36,9 +46,8 @@ index 46183b0..d1d27d8 100644 - diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java -index 53a84cb..3af4b6b 100644 +index 53a84cbdd53..2ed43542a03 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressionMode.java @@ -22,6 +22,8 @@ import java.util.zip.DataFormatException; @@ -83,7 +92,7 @@ index 53a84cb..3af4b6b 100644 /** * Create a new {@link Compressor} instance. -@@ -296,4 +319,111 @@ public abstract class CompressionMode { +@@ -296,4 +319,105 @@ public abstract class CompressionMode { } @@ -109,25 +118,19 @@ index 53a84cb..3af4b6b 100644 + return; + } + final int compressedLength = in.readVInt(); -+ // pad with extra "dummy byte": see javadocs for using Inflater(true) -+ // we do it for compliance, but it's unnecessary for years in zlib. -+ final int paddedLength = compressedLength; -+ compressed = ArrayUtil.grow(compressed, paddedLength + 1); ++ compressed = new byte[compressedLength]; + in.readBytes(compressed, 0, compressedLength); -+ compressed[compressedLength] = 0; // explicitly set dummy byte to 0 + + final QatDecompressorJNI decompressor = new QatDecompressorJNI(); + + try { -+ // extra "dummy byte" -+ decompressor.setInput(compressed, 0, paddedLength); ++ decompressor.setInput(compressed, 0, compressedLength); + bytes.offset = bytes.length = 0; + bytes.bytes = ArrayUtil.grow(bytes.bytes, originalLength); + try { + bytes.length = decompressor.decompress(bytes.bytes, bytes.length, originalLength); + } catch (Error e) { -+ String s = e.getMessage(); -+ System.out.println(e.getMessage()); ++ throw new Error(e); + } + + if (!decompressor.finished()) { @@ -169,7 +172,7 @@ index 53a84cb..3af4b6b 100644 + compressor.finish(); + + int totalCount = 0; -+ for (; ; ) { ++ while (!compressor.finished() ) { + final int count = compressor.compress(compressed, totalCount, compressed.length - totalCount); + totalCount += count; + assert totalCount <= compressed.length; @@ -196,7 +199,7 @@ index 53a84cb..3af4b6b 100644 + } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java -index fdfba5b..7c61e50 100644 +index fdfba5b7677..7c61e50f87e 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50StoredFieldsFormat.java @@ -145,7 +145,9 @@ public final class Lucene50StoredFieldsFormat extends StoredFieldsFormat { @@ -220,7 +223,7 @@ index fdfba5b..7c61e50 100644 } } diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java -index 62d06d8..6e4f1a2 100644 +index 62d06d8f0f6..6e4f1a2389c 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/AbstractTestCompressionMode.java @@ -20,15 +20,16 @@ package org.apache.lucene.codecs.compressing; @@ -273,18 +276,9 @@ index 62d06d8..6e4f1a2 100644 } } -@@ -127,7 +141,7 @@ public abstract class AbstractTestCompressionMode extends LuceneTestCase { - } - - public void testShortSequence() throws IOException { -- test(new byte[] { (byte) random().nextInt(256) }); -+ test(new byte[] { (byte) random().nextInt(256)}); - } - - public void testIncompressible() throws IOException { diff --git a/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestQatCompressionDecompressionMode.java b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestQatCompressionDecompressionMode.java new file mode 100644 -index 0000000..9bacf20 +index 00000000000..9bacf20caa9 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/compressing/TestQatCompressionDecompressionMode.java @@ -0,0 +1,26 @@ @@ -316,7 +310,7 @@ index 0000000..9bacf20 +} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatQatCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatQatCompression.java new file mode 100644 -index 0000000..e361fcd +index 00000000000..e361fcdec08 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50StoredFieldsFormatQatCompression.java @@ -0,0 +1,80 @@ @@ -401,17 +395,9 @@ index 0000000..e361fcd + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java -index 4fd5e16..b06d86a 100644 +index 4fd5e16cef8..b06d86af5b4 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/CompressingCodec.java -@@ -26,6 +26,7 @@ import org.apache.lucene.util.TestUtil; - - import com.carrotsearch.randomizedtesting.generators.RandomNumbers; - -+ - /** - * A codec that uses {@link CompressingStoredFieldsFormat} for its stored - * fields and delegates to the default codec for everything else. @@ -36,22 +37,26 @@ public abstract class CompressingCodec extends FilterCodec { * Create a random instance. */ @@ -445,15 +431,9 @@ index 4fd5e16..b06d86a 100644 } } - /** -- * Creates a random {@link CompressingCodec} that is using an empty segment -+ * Creates a random {@link CompressingCodec} that is using an empty segment - * suffix - */ - public static CompressingCodec randomInstance(Random random) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/QatCompressionCompressingCodec.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/QatCompressionCompressingCodec.java new file mode 100644 -index 0000000..ffaaac7 +index 00000000000..dfc322cbf79 --- /dev/null +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/compressing/QatCompressionCompressingCodec.java @@ -0,0 +1,41 @@ @@ -495,11 +475,11 @@ index 0000000..ffaaac7 + public QatCompressionCompressingCodec() { + // we don't worry about zlib block overhead as it's + // not bad and try to save space instead: -+ this(61440, 512, false, 1024); ++ this(60*1024, 512, false, 1024); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java -index 7acee87..6c5368c 100644 +index 7acee871f59..54b042705b2 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java @@ -60,6 +60,7 @@ import org.apache.lucene.store.Directory; @@ -519,7 +499,7 @@ index 7acee87..6c5368c 100644 /** diff --git a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec -index 282f5dd..292c393 100644 +index 282f5dd20c8..292c3931963 100644 --- a/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -19,3 +19,4 @@ org.apache.lucene.codecs.compressing.FastCompressingCodec @@ -528,7 +508,7 @@ index 282f5dd..292c393 100644 org.apache.lucene.codecs.compressing.dummy.DummyCompressingCodec +org.apache.lucene.codecs.compressing.QatCompressionCompressingCodec diff --git a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java -index cf1c6a3..83c9737 100644 +index cf1c6a30a39..83c9737cb78 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java +++ b/lucene/test-framework/src/test/org/apache/lucene/codecs/compressing/TestCompressingTermVectorsFormat.java @@ -38,7 +38,9 @@ import org.apache.lucene.index.TermsEnum.SeekStatus; @@ -542,7 +522,7 @@ index cf1c6a3..83c9737 100644 @Override diff --git a/lucene/tools/build.xml b/lucene/tools/build.xml -index b245dce..ab887cc 100644 +index b245dce2e9a..ab887ccf064 100644 --- a/lucene/tools/build.xml +++ b/lucene/tools/build.xml @@ -29,10 +29,13 @@ @@ -562,7 +542,7 @@ index b245dce..ab887cc 100644 diff --git a/lucene/tools/junit4/tests.policy b/lucene/tools/junit4/tests.policy -index 7494981..c31be98 100644 +index 74949813b7c..c31be9867c0 100644 --- a/lucene/tools/junit4/tests.policy +++ b/lucene/tools/junit4/tests.policy @@ -28,6 +28,12 @@ grant { From 0e8abee404f7a059b9b34b838a6a4aee3cb48788 Mon Sep 17 00:00:00 2001 From: zhangjie Date: Sat, 28 Mar 2020 00:22:31 +0800 Subject: [PATCH 12/13] small changes according comments --- lucene_qat_wrapper/pom.xml | 2 +- .../com/intel/qat/jni/QatCompressorJNI.java | 25 ++++++++----------- .../com/intel/qat/jni/QatDecompressorJNI.java | 18 ++++++------- .../intel/qat/util/QatNativeCodeLoader.java | 6 +---- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/lucene_qat_wrapper/pom.xml b/lucene_qat_wrapper/pom.xml index 3322df6..1c9fd3d 100644 --- a/lucene_qat_wrapper/pom.xml +++ b/lucene_qat_wrapper/pom.xml @@ -33,7 +33,7 @@ 3.0.1 3.0.0 2.10.4 - + 3.0.0 3.0.0 2.5.2 2.8.2 diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java index 8fb4abb..e6940bf 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java @@ -28,14 +28,10 @@ import java.nio.Buffer; import java.nio.ByteBuffer; - /** - * @author root - * @date 12/3/19 12:39 PM * @decsription: A {@link Compressor} based on the qat compression algorithm. * @link qat-parent */ - public class QatCompressorJNI { private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class); @@ -46,6 +42,16 @@ public class QatCompressorJNI { private static Class clazz = QatCompressorJNI.class; private static boolean nativeQatLoaded = false; + private volatile int directBufferSize; + private volatile Buffer compressedDirectBuf = null; + private volatile int uncompressedDirectBufLen; + private volatile Buffer uncompressedDirectBuf = null; + private volatile byte[] userBuf = null; + private volatile int userBufOff = 0, userBufLen = 0; + private volatile boolean finish, finished; + private volatile long bytesRead = 0L; + private volatile long bytesWritten = 0L; + static { if (QatNativeCodeLoader.isNativeCodeLoaded() && QatNativeCodeLoader.buildSupportsQat()) { @@ -76,16 +82,6 @@ public class QatCompressorJNI { } } - private int directBufferSize; - private Buffer compressedDirectBuf = null; - private int uncompressedDirectBufLen; - private Buffer uncompressedDirectBuf = null; - private byte[] userBuf = null; - private int userBufOff = 0, userBufLen = 0; - private boolean finish, finished; - private long bytesRead = 0L; - private long bytesWritten = 0L; - /** * Creates a new compressor. * @@ -342,6 +338,7 @@ public synchronized long getBytesWritten() { */ public synchronized void end() { + //do nothing } private native int compressBytesDirect(); diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java index 3e109a2..b7ecf96 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -27,8 +27,6 @@ import java.nio.ByteBuffer; /** - * @author root - * @date 12/3/19 12:47 PM * @decsription: A {@link Decompressor} based on the qat compression algorithm. It's not thread-safe. * @link qat-parent */ @@ -42,6 +40,14 @@ public class QatDecompressorJNI { private static Class clazz = QatDecompressorJNI.class; private static boolean nativeQatLoaded = false; + private int directBufferSize; + private Buffer compressedDirectBuf = null; + private int compressedDirectBufLen; + private Buffer uncompressedDirectBuf = null; + private byte[] userBuf = null; + private int userBufOff = 0, userBufLen = 0; + private boolean finished; + static { if (QatNativeCodeLoader.isNativeCodeLoaded() && QatNativeCodeLoader.buildSupportsQat()) { @@ -54,14 +60,6 @@ public class QatDecompressorJNI { } } - private int directBufferSize; - private Buffer compressedDirectBuf = null; - private int compressedDirectBufLen; - private Buffer uncompressedDirectBuf = null; - private byte[] userBuf = null; - private int userBufOff = 0, userBufLen = 0; - private boolean finished; - /** * Creates a new decompressor. * diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java index 86deb02..691396f 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/util/QatNativeCodeLoader.java @@ -20,15 +20,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; + /** * A helper to load the native qat code i.e. libqat.so. * This handles the fallback to either the bundled libhadoop-Linux-i386-32.so * or the default java implementations where appropriate. - * */ - public class QatNativeCodeLoader { - //private static final Logger LOG = LoggerFactory.getLogger(QatNativeCodeLoader.class); private static final Logger LOG = LogManager.getLogger(QatNativeCodeLoader.class); private static boolean nativeCodeLoaded = false; @@ -43,11 +41,9 @@ public class QatNativeCodeLoader { nativeCodeLoaded = true; } catch (Throwable t) { // Ignore failure to load - if(LOG.isDebugEnabled()) { LOG.debug("Failed to load native-qat with error: " + t); LOG.debug("java.library.path=" + System.getProperty("java.library.path")); - } } if (!nativeCodeLoaded) { From bc68800e6a79922fddea4cb179dd977f55504efb Mon Sep 17 00:00:00 2001 From: Zhang jie1 Date: Tue, 31 Mar 2020 23:03:14 +0800 Subject: [PATCH 13/13] change directBuffSize to solve Full GC --- .../src/main/java/com/intel/qat/jni/QatCompressorJNI.java | 2 +- .../src/main/java/com/intel/qat/jni/QatDecompressorJNI.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java index e6940bf..cbede03 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatCompressorJNI.java @@ -35,7 +35,7 @@ public class QatCompressorJNI { private static final Logger LOG = LogManager.getLogger(QatCompressorJNI.class); - private static final int DEFAULT_DIRECT_BUFFER_SIZE = 640 * 1024; + private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; // HACK - Use this as a global lock in the JNI layer @SuppressWarnings({"rawtypes"}) diff --git a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java index b7ecf96..48ccd3e 100644 --- a/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java +++ b/lucene_qat_wrapper/src/main/java/com/intel/qat/jni/QatDecompressorJNI.java @@ -33,7 +33,7 @@ public class QatDecompressorJNI { private static final Logger LOG = LogManager.getLogger(QatDecompressorJNI.class); - private static final int DEFAULT_DIRECT_BUFFER_SIZE = 640 * 1024; + private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024; // HACK - Use this as a global lock in the JNI layer @SuppressWarnings({"rawtypes"})