Skip to content

Commit 1cfc00b

Browse files
tinder-igorsokolovtinder-igorsokolov
committed
added serialization / deserialization to Xor8, Xor16, XorBinaryFuse8, XorBinaryFuse32
1 parent 69abf1a commit 1cfc00b

File tree

5 files changed

+294
-57
lines changed

5 files changed

+294
-57
lines changed

‎fastfilter/src/main/java/org/fastfilter/xor/Xor16.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.fastfilter.xor;
22

3+
import java.nio.ByteBuffer;
4+
35
import org.fastfilter.Filter;
46
import org.fastfilter.utils.Hash;
57

@@ -143,4 +145,55 @@ private int fingerprint(long hash) {
143145
return (int) (hash & ((1 << BITS_PER_FINGERPRINT) - 1));
144146
}
145147

148+
private Xor16(int blockLength, int bitCount, long seed, short[] fingerprints) {
149+
this.blockLength = blockLength;
150+
this.bitCount = bitCount;
151+
this.seed = seed;
152+
this.fingerprints = fingerprints;
153+
}
154+
155+
@Override
156+
public int getSerializedSize() {
157+
return Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Short.BYTES;
158+
}
159+
160+
@Override
161+
public void serialize(ByteBuffer buffer) {
162+
if (buffer.remaining() < getSerializedSize()) {
163+
throw new IllegalArgumentException("Buffer too small");
164+
}
165+
166+
buffer.putInt(blockLength);
167+
buffer.putLong(seed);
168+
buffer.putInt(fingerprints.length);
169+
for (final short fp : fingerprints) {
170+
buffer.putShort(fp);
171+
}
172+
}
173+
174+
public static Xor16 deserialize(ByteBuffer buffer) {
175+
// Check minimum size for header (1 int + 1 long + 1 int for length)
176+
if (buffer.remaining() < Integer.BYTES + Long.BYTES + Integer.BYTES) {
177+
throw new IllegalArgumentException("Buffer too small");
178+
}
179+
180+
final int blockLength = buffer.getInt();
181+
final long seed = buffer.getLong();
182+
183+
final int len = buffer.getInt();
184+
185+
// Check if buffer has enough bytes for all fingerprints
186+
if (buffer.remaining() < len * Short.BYTES) {
187+
throw new IllegalArgumentException("Buffer too small");
188+
}
189+
190+
final short[] fingerprints = new short[len];
191+
for (int i = 0; i < len; i++) {
192+
fingerprints[i] = buffer.getShort();
193+
}
194+
195+
final int bitCount = len * BITS_PER_FINGERPRINT;
196+
197+
return new Xor16(blockLength, bitCount, seed, fingerprints);
198+
}
146199
}

‎fastfilter/src/main/java/org/fastfilter/xor/Xor8.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.fastfilter.xor;
22

33
import java.io.*;
4+
import java.nio.ByteBuffer;
45

56
import org.fastfilter.Filter;
67
import org.fastfilter.utils.Hash;
@@ -187,4 +188,51 @@ public Xor8(InputStream in) {
187188
}
188189
}
189190

191+
private Xor8(int size, long seed, byte[] fingerprints) {
192+
this.size = size;
193+
this.arrayLength = getArrayLength(size);
194+
this.bitCount = arrayLength * BITS_PER_FINGERPRINT;
195+
this.blockLength = arrayLength / HASHES;
196+
this.seed = seed;
197+
this.fingerprints = fingerprints;
198+
}
199+
200+
@Override
201+
public int getSerializedSize() {
202+
return Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Byte.BYTES;
203+
}
204+
205+
@Override
206+
public void serialize(ByteBuffer buffer) {
207+
if (buffer.remaining() < getSerializedSize()) {
208+
throw new IllegalArgumentException("Buffer too small");
209+
}
210+
211+
buffer.putInt(size);
212+
buffer.putLong(seed);
213+
buffer.putInt(fingerprints.length);
214+
buffer.put(fingerprints);
215+
}
216+
217+
public static Xor8 deserialize(ByteBuffer buffer) {
218+
// Check minimum size for header (1 int + 1 long + 1 int for length)
219+
if (buffer.remaining() < Integer.BYTES + Long.BYTES + Integer.BYTES) {
220+
throw new IllegalArgumentException("Buffer too small");
221+
}
222+
223+
final int size = buffer.getInt();
224+
final long seed = buffer.getLong();
225+
226+
final int len = buffer.getInt();
227+
228+
// Check if buffer has enough bytes for all fingerprints
229+
if (buffer.remaining() < len * Byte.BYTES) {
230+
throw new IllegalArgumentException("Buffer too small");
231+
}
232+
233+
final byte[] fingerprints = new byte[len];
234+
buffer.get(fingerprints);
235+
236+
return new Xor8(size, seed, fingerprints);
237+
}
190238
}

‎fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse32.java‎

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.fastfilter.xor;
22

3+
import java.nio.ByteBuffer;
34
import java.util.Arrays;
45

56
import org.fastfilter.Filter;
@@ -20,19 +21,25 @@ public class XorBinaryFuse32 implements Filter {
2021
private final int[] fingerprints;
2122
private long seed;
2223

23-
public XorBinaryFuse32(int segmentCount, int segmentLength) {
24+
private XorBinaryFuse32(int segmentCount, int segmentLength, long seed, int[] fingerprints) {
2425
if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) {
2526
throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength);
2627
}
2728
if (segmentCount <= 0) {
2829
throw new IllegalArgumentException("Illegal segment count: " + segmentCount);
2930
}
30-
this.segmentLength = segmentLength;
31+
3132
this.segmentCount = segmentCount;
32-
this.segmentLengthMask = segmentLength - 1;
3333
this.segmentCountLength = segmentCount * segmentLength;
34-
this.arrayLength = (segmentCount + ARITY - 1) * segmentLength;
35-
this.fingerprints = new int[arrayLength];
34+
this.segmentLength = segmentLength;
35+
this.segmentLengthMask = segmentLength - 1;
36+
this.arrayLength = fingerprints.length;
37+
this.fingerprints = fingerprints;
38+
this.seed = seed;
39+
}
40+
41+
public XorBinaryFuse32(int segmentCount, int segmentLength) {
42+
this(segmentCount, segmentLength, 0L, new int[(segmentCount + ARITY - 1) * segmentLength]);
3643
}
3744

3845
public long getBitCount() {
@@ -261,4 +268,51 @@ private int fingerprint(long hash) {
261268
return (int) (hash ^ (hash >>> 32));
262269
}
263270

271+
@Override
272+
public int getSerializedSize() {
273+
return 2 * Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Integer.BYTES;
274+
}
275+
276+
@Override
277+
public void serialize(ByteBuffer buffer) {
278+
if (buffer.remaining() < getSerializedSize()) {
279+
throw new IllegalArgumentException("Buffer too small");
280+
}
281+
282+
buffer.putInt(segmentLength);
283+
buffer.putInt(segmentCountLength);
284+
buffer.putLong(seed);
285+
buffer.putInt(fingerprints.length);
286+
for (final int fp : fingerprints) {
287+
buffer.putInt(fp);
288+
}
289+
}
290+
291+
public static XorBinaryFuse32 deserialize(ByteBuffer buffer) {
292+
// Check minimum size for header (2 ints + 1 long + 1 int for length)
293+
if (buffer.remaining() < 2 * Integer.BYTES + Long.BYTES + Integer.BYTES) {
294+
throw new IllegalArgumentException("Buffer too small");
295+
}
296+
297+
final int segmentLength = buffer.getInt();
298+
final int segmentCountLength = buffer.getInt();
299+
final long seed = buffer.getLong();
300+
301+
final int len = buffer.getInt();
302+
303+
// Check if buffer has enough bytes for all fingerprints
304+
if (buffer.remaining() < len * Integer.BYTES) {
305+
throw new IllegalArgumentException("Buffer too small");
306+
}
307+
308+
final int[] fingerprints = new int[len];
309+
for (int i = 0; i < len; i++) {
310+
fingerprints[i] = buffer.getInt();
311+
}
312+
313+
// Calculate segmentCount from segmentCountLength and segmentLength
314+
final int segmentCount = segmentCountLength / segmentLength;
315+
316+
return new XorBinaryFuse32(segmentCount, segmentLength, seed, fingerprints);
317+
}
264318
}

‎fastfilter/src/main/java/org/fastfilter/xor/XorBinaryFuse8.java‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.fastfilter.xor;
22

3+
import java.nio.ByteBuffer;
34
import java.util.Arrays;
45

56
import org.fastfilter.Filter;
@@ -20,19 +21,25 @@ public class XorBinaryFuse8 implements Filter {
2021
private final byte[] fingerprints;
2122
private long seed;
2223

23-
public XorBinaryFuse8(int segmentCount, int segmentLength) {
24+
private XorBinaryFuse8(int segmentCount, int segmentLength, long seed, byte[] fingerprints) {
2425
if (segmentLength < 0 || Integer.bitCount(segmentLength) != 1) {
2526
throw new IllegalArgumentException("Segment length needs to be a power of 2, is " + segmentLength);
2627
}
2728
if (segmentCount <= 0) {
2829
throw new IllegalArgumentException("Illegal segment count: " + segmentCount);
2930
}
30-
this.segmentLength = segmentLength;
31+
3132
this.segmentCount = segmentCount;
32-
this.segmentLengthMask = segmentLength - 1;
3333
this.segmentCountLength = segmentCount * segmentLength;
34-
this.arrayLength = (segmentCount + ARITY - 1) * segmentLength;
35-
this.fingerprints = new byte[arrayLength];
34+
this.segmentLength = segmentLength;
35+
this.segmentLengthMask = segmentLength - 1;
36+
this.arrayLength = fingerprints.length;
37+
this.fingerprints = fingerprints;
38+
this.seed = seed;
39+
}
40+
41+
public XorBinaryFuse8(int segmentCount, int segmentLength) {
42+
this(segmentCount, segmentLength, 0L, new byte[(segmentCount + ARITY - 1) * segmentLength]);
3643
}
3744

3845
public long getBitCount() {
@@ -261,4 +268,47 @@ private byte fingerprint(long hash) {
261268
return (byte) hash;
262269
}
263270

271+
@Override
272+
public int getSerializedSize() {
273+
return 2 * Integer.BYTES + Long.BYTES + Integer.BYTES + fingerprints.length * Byte.BYTES;
274+
}
275+
276+
@Override
277+
public void serialize(ByteBuffer buffer) {
278+
if (buffer.remaining() < getSerializedSize()) {
279+
throw new IllegalArgumentException("Buffer too small");
280+
}
281+
282+
buffer.putInt(segmentLength);
283+
buffer.putInt(segmentCountLength);
284+
buffer.putLong(seed);
285+
buffer.putInt(fingerprints.length);
286+
buffer.put(fingerprints);
287+
}
288+
289+
public static XorBinaryFuse8 deserialize(ByteBuffer buffer) {
290+
// Check minimum size for header (2 ints + 1 long + 1 int for length)
291+
if (buffer.remaining() < 2 * Integer.BYTES + Long.BYTES + Integer.BYTES) {
292+
throw new IllegalArgumentException("Buffer too small");
293+
}
294+
295+
final int segmentLength = buffer.getInt();
296+
final int segmentCountLength = buffer.getInt();
297+
final long seed = buffer.getLong();
298+
299+
final int len = buffer.getInt();
300+
301+
// Check if buffer has enough bytes for all fingerprints
302+
if (buffer.remaining() < len * Byte.BYTES) {
303+
throw new IllegalArgumentException("Buffer too small");
304+
}
305+
306+
final byte[] fingerprints = new byte[len];
307+
buffer.get(fingerprints);
308+
309+
// Calculate segmentCount from segmentCountLength and segmentLength
310+
final int segmentCount = segmentCountLength / segmentLength;
311+
312+
return new XorBinaryFuse8(segmentCount, segmentLength, seed, fingerprints);
313+
}
264314
}

0 commit comments

Comments
 (0)