/** * Creates a new big-array big list with given capacity. * * @param capacity * the initial capacity of the array list (may be 0). */ public LongBigArrayBigList(final long capacity) { if (capacity < 0) throw new IllegalArgumentException("Initial capacity (" + capacity + ") is negative"); if (capacity == 0) a = LongBigArrays.EMPTY_BIG_ARRAY; else a = LongBigArrays.newBigArray(capacity); } /**
private void readObject(java.io.ObjectInputStream s) throws java.io.IOException, ClassNotFoundException { s.defaultReadObject(); a = LongBigArrays.newBigArray(size); for (int i = 0; i < size; i++) LongBigArrays.set(a, i, s.readLong()); } }
/** Stores the identity permutation in a new big array of given length. * * @param n the size of the array. * @return a new array of length <code>n</code>, filled with the identity permutation. */ public static long[][] identity(long n) { return identity(LongBigArrays.newBigArray(n)); } }
/** * Turns a standard array into a big array. * * <p> * Note that the returned big array might contain as a segment the original * array. * * @param array * an array. * @return a new big array with the same length and content of {@code array}. */ public static long[][] wrap(final long[] array) { if (array.length == 0) return EMPTY_BIG_ARRAY; if (array.length <= SEGMENT_SIZE) return new long[][]{array}; final long[][] bigArray = newBigArray(array.length); for (int i = 0; i < bigArray.length; i++) System.arraycopy(array, (int) start(i), bigArray[i], 0, bigArray[i].length); return bigArray; } /**
/** Loads elements from a file given by a {@link File} object, storing them in a new big array. * * <p>Note that the length of the returned big array will be computed * dividing the specified file size by the number of bytes used to * represent each element. * * @param file a file. * @return a big array filled with the content of the specified file. */ public static long[][] loadLongsBig(final File file) throws IOException { final FileInputStream fis = new FileInputStream(file); final long length = fis.getChannel().size() / (Long.SIZE / 8); final long[][] array = LongBigArrays.newBigArray(length); final DataInputStream dis = new DataInputStream(new FastBufferedInputStream(fis)); for(int i = 0; i < array.length; i++) { final long[] t = array[i]; final int l = t.length; for(int d = 0; d < l; d++) t[d] = dis.readLong(); } dis.close(); return array; } /** Loads elements from a file given by a filename, storing them in a new big array.
/** Computes the inverse of a permutation expressed * as a {@linkplain BigArrays big array} of <var>n</var> distinct long integers in [0 .. <var>n</var>) * and stores the result in a new big array. * * <p><strong>Warning</strong>: if <code>perm</code> is not a permutation, * essentially anything can happen. * * @param perm the permutation to be inverted. * @return a new big array containing the inverse permutation. */ public static long[][] invertPermutation(long[][] perm) { return invertPermutation(perm, LongBigArrays.newBigArray(LongBigArrays.length(perm))); }
/** Commodity method for loading a big list of binary longs with specified endianness into a * {@linkplain LongBigArrays long big array}. * * @param filename the file containing the longs. * @param byteOrder the endianness of the longs. * @return a big list of longs containing the longs in <code>filename</code>. */ public static LongBigArrayBigList loadLongBigList(final CharSequence filename, final ByteOrder byteOrder) throws IOException { final long length = new File(filename.toString()).length() / (Long.SIZE / Byte.SIZE); @SuppressWarnings("resource") final ReadableByteChannel channel = new FileInputStream(filename.toString()).getChannel(); final ByteBuffer byteBuffer = ByteBuffer.allocateDirect(64 * 1024).order(byteOrder); final LongBuffer longBuffer = byteBuffer.asLongBuffer(); final long[][] array = LongBigArrays.newBigArray(length); long pos = 0; while (channel.read(byteBuffer) > 0) { byteBuffer.flip(); final int remainingLongs = byteBuffer.remaining() / (Long.SIZE / Byte.SIZE); longBuffer.clear(); longBuffer.limit(remainingLongs); longBuffer.get(array[BigArrays.segment(pos)], BigArrays.displacement(pos), remainingLongs); pos += remainingLongs; byteBuffer.clear(); } channel.close(); return LongBigArrayBigList.wrap(array); }
/** * Returns a copy of a portion of a big array. * * @param array * a big array. * @param offset * the first element to copy. * @param length * the number of elements to copy. * @return a new big array containing {@code length} elements of {@code array} * starting at {@code offset}. */ public static long[][] copy(final long[][] array, final long offset, final long length) { ensureOffsetLength(array, offset, length); final long[][] a = newBigArray(length); copy(array, offset, a, 0, length); return a; } /**
/** * Creates a new hash big set. * * <p> * The actual table size will be the least power of two greater than * {@code expected}/{@code f}. * * @param expected * the expected number of elements in the set. * @param f * the load factor. */ public LongOpenHashBigSet(final long expected, final float f) { if (f <= 0 || f > 1) throw new IllegalArgumentException("Load factor must be greater than 0 and smaller than or equal to 1"); if (n < 0) throw new IllegalArgumentException("The expected number of elements must be nonnegative"); this.f = f; minN = n = bigArraySize(expected, f); maxFill = maxFill(n, f); key = LongBigArrays.newBigArray(n); initMasks(); } /**
private void readObject(java.io.ObjectInputStream s) throws java.io.IOException, ClassNotFoundException { s.defaultReadObject(); n = bigArraySize(size, f); maxFill = maxFill(n, f); final long[][] key = this.key = LongBigArrays.newBigArray(n); initMasks(); long h; long k; int base, displ; for (long i = size; i-- != 0;) { k = s.readLong(); if (((k) == (0))) containsNull = true; else { h = it.unimi.dsi.fastutil.HashCommon.mix((k)); if (!((key[base = (int) ((h & mask) >>> BigArrays.SEGMENT_SHIFT)][displ = (int) (h & segmentMask)]) == (0))) while (!((key[base = (base + ((displ = (displ + 1) & segmentMask) == 0 ? 1 : 0)) & baseMask][displ]) == (0))); key[base][displ] = k; } } if (ASSERTS) checkTable(); } private void checkTable() {
/** Utility method to load a compressed offset file into a list. * * @param in the input bit stream providing the offsets (see {@link BitStreamIndexWriter}). * @param T the number of terms indexed. * @return a list of longs backed by an array; the list has * an additional final element of index <code>T</code> that gives the number * of bytes of the index file. */ public static LongBigList readOffsets( final InputBitStream in, final long T ) throws IOException { final long[][] offset = LongBigArrays.newBigArray( T + 1 ); LOGGER.debug( "Loading offsets..." ); long prev; LongBigArrays.set( offset, 0, prev = in.readLongGamma() ); for( int i = 0; i < T; i++ ) LongBigArrays.set( offset, i + 1, prev = in.readLongGamma() + prev ); LOGGER.debug( "Completed." ); return LongBigArrayBigList.wrap( offset ); }
/** Utility method to load a compressed offset file into a list. * * @param in the input bit stream providing the offsets (see {@link BitStreamIndexWriter}). * @param T the number of terms indexed. * @return a list of longs backed by an array; the list has * an additional final element of index <code>T</code> that gives the number * of bytes of the index file. */ public static LongBigList readOffsets( final InputBitStream in, final long T ) throws IOException { final long[][] offset = LongBigArrays.newBigArray( T + 1 ); LOGGER.debug( "Loading offsets..." ); long prev; LongBigArrays.set( offset, 0, prev = in.readLongGamma() ); for( int i = 0; i < T; i++ ) LongBigArrays.set( offset, i + 1, prev = in.readLongGamma() + prev ); LOGGER.debug( "Completed." ); return LongBigArrayBigList.wrap( offset ); }
/** * Creates a new hash big set. * * <p>The actual table size will be the least power of two greater than * <code>expected</code>/<code>f</code>. * * @param expected the expected number of elements in the set. * @param f the load factor. */ public Long2IntOpenHashBigMap(final long expected, final float f) { if (f <= 0 || f > 1) throw new IllegalArgumentException("Load factor must be greater than 0 and smaller than or equal to 1"); if (n < 0) throw new IllegalArgumentException("The expected number of elements must be nonnegative"); this.f = f; n = bigArraySize(expected, f); maxFill = maxFill(n, f); key = LongBigArrays.newBigArray(n); value = IntBigArrays.newBigArray(n); used = BooleanBigArrays.newBigArray(n); initMasks(); }
/** Utility method to load a compressed offset file into a list. * * @param filename the file containing the offsets (see {@link BitStreamIndexWriter}). * @param T the number of terms indexed. * @return a list of longs backed by an array; the list has * an additional final element of index <code>T</code> that gives the number * of bytes of the index file. */ public static LongBigList readOffsets( final CharSequence filename, final long T ) throws IOException { final InputBitStream in = new InputBitStream( filename.toString() ); final long[][] offset = LongBigArrays.newBigArray( T + 1 ); LOGGER.debug( "Loading offsets..." ); long prev; LongBigArrays.set( offset, 0, prev = in.readLongGamma() ); for( int i = 0; i < T; i++ ) LongBigArrays.set( offset, i + 1, prev = in.readLongGamma() + prev ); in.close(); LOGGER.debug( "Completed." ); return LongBigArrayBigList.wrap( offset ); }
/** Generates a 64-bit signatures big array using a given function and string sequence. * * <p>The resulting big array can be saved using {@link BinIO#storeLongs(long[][], CharSequence)} * or similar {@link BinIO} methods. * * @param iterator an iterator over a list of strings. * @param function the function to be signed. * @param pl a progress logger, or {@code null}. * @return a big array of 64-bit signatures. */ public static long[][] sign(final Iterator<? extends CharSequence> iterator, final Object2LongFunction<? extends CharSequence> function , final ProgressLogger pl) { final long n = function instanceof Size64 ? ((Size64)function).size64() : function .size(); final long[][] signature = LongBigArrays.newBigArray(n); if (pl != null) { pl.expectedUpdates = n; pl.start("Signing..."); } CharSequence s; for(long i = 0; i < n; i++) { s = iterator.next(); LongBigArrays.set(signature, function .getLong(s), signature(s)); if (pl != null) pl.lightUpdate(); } if (iterator.hasNext()) throw new IllegalStateException("Iterator provides more than " + n + " elements"); if (pl != null) pl.done(); return signature; }
final long newKey[][] = LongBigArrays.newBigArray(newN); final long mask = newN - 1; // Note that this is used by the hashing macro final int newSegmentMask = newKey[0].length - 1;
final int[][] value = this.value; final boolean newUsed[][] = BooleanBigArrays.newBigArray(newN); final long newKey[][] = LongBigArrays.newBigArray(newN); final int newValue[][] = IntBigArrays.newBigArray(newN); final long newMask = newN - 1;