private void cleanUpCache(Futures fs) { final Key[] cacheKeys = KeySnapshot.globalSnapshot().filter(new KeySnapshot.KVFilter() { @Override public boolean filter(KeySnapshot.KeyInfo k) { return Value.isSubclassOf(k._type, DeepWaterImageIterator.IcedImage.class) && k._key.toString().contains(CACHE_MARKER) || Value.isSubclassOf(k._type, DeepWaterDatasetIterator.IcedRow.class) && k._key.toString().contains(CACHE_MARKER); } }).keys(); if (fs==null) fs = new Futures(); for (Key k : cacheKeys) DKV.remove(k, fs); fs.blockForPending(); }
@Override protected void setupLocal() { _model_mem_size = 0; for (int i=0; i< trees_so_far; ++i) { Key<CompressedTree>[] per_class = _treeKeys[i]; for (int j=0; j<per_class.length; ++j) { if (per_class[j] == null) continue; if (!per_class[j].home()) continue; // only look at homed tree keys _model_mem_size += DKV.get(per_class[j])._max; } } }
public T invokeOnAllNodes() { H2O cloud = H2O.CLOUD; Key[] args = new Key[cloud.size()]; String skey = "RunOnAll"+Key.rand(); for( int i = 0; i < args.length; ++i ) args[i] = Key.make(skey,(byte)0,Key.DFJ_INTERNAL_USER,cloud._memary[i]); invoke(args); for( Key arg : args ) DKV.remove(arg); return self(); }
@Override public final void lonCompletion( CountedCompleter caller ) { Futures fs = new Futures(); for(Vec v:vs) if(v._key.home()) DKV.put(v._key,v,fs); fs.blockForPending(); } @Override public void reduce(DRemoteTask drt){}
@Override public final void compute2(){ if(_key.home()){ Value val = H2O.get(_key); if(val != null) { V v = val.get(); map(v); } tryComplete(); } else new RPC(_key.home_node(),this).addCompleter(this).call(); } // onCompletion must be empty here, may be invoked twice (on remote and local)
public static void delete( Key k, Key job_key ) { if( k == null ) return; Value val = DKV.get(k); if( val == null ) return; // Or just nothing there to delete if( !val.isLockable() ) UKV.remove(k); // Simple things being deleted else ((Lockable)val.get()).delete(job_key,0.0f); // Lockable being deleted } // Will fail if locked by anybody.
public static void read_lock( Key k, Key job_key ) { Value val = DKV.get(k); if( val.isLockable() ) ((Lockable)val.get()).read_lock(job_key); // Lockable being locked } public void read_lock( Key job_key ) {
public ChunkProgressJob(long chunksTotal, Key destinationKey) { destination_key = destinationKey; _progress = Key.make(Key.make()._kb, (byte) 0, Key.DFJ_INTERNAL_USER, destinationKey.home_node()); UKV.put(_progress, new ChunkProgress(chunksTotal)); }
public <T extends Freezable> T getFreezable() { touch(); Freezable pojo = _pojo; // Read once! if( pojo != null ) return (T)pojo; pojo = TypeMap.newFreezable(_type); pojo.read(new AutoBuffer(memOrLoad())); if( pojo instanceof Iced ) ((Iced)pojo).init(_key); return (T)(_pojo = pojo); }
public <T extends Iced> T get() { touch(); Iced pojo = (Iced)_pojo; // Read once! if( pojo != null ) return (T)pojo; pojo = TypeMap.newInstance(_type); pojo.read(new AutoBuffer(memOrLoad())); pojo.init(_key); return (T)(_pojo = pojo); } public <T extends Freezable> T get(Class<T> fc) {
private static <T extends Keyed> Key<T> publish(Keyed<T> keyed) { Scope.track_generic(keyed); DKV.put(keyed); return keyed._key; }
AutoBuffer call(AutoBuffer ab) { int tnum = ab.getTask(); RPC<?> t = ab._h2o.taskGet(tnum); assert t== null || t._tasknum == tnum; if( t != null ) t.response(ab); // Do the 2nd half of this task, includes ACKACK else ab.close(); // Else forgotten task, but still must ACKACK return new AutoBuffer(ab._h2o).putTask(UDP.udp.ackack.ordinal(),tnum); }
@Override public void compute2() { _ab.getPort(); // skip past the port if( _ctrl <= UDP.udp.ack.ordinal() ) UDP.udp.UDPS[_ctrl]._udp.call(_ab).close(); else RPC.remote_exec(_ab); tryComplete(); } /** Exceptional completion path; mostly does printing if the exception was
public static String fetchClazz(int id) { String clazz = RPC.call(H2O.CLOUD.leader(), new FetchClazz(id)).get()._clazz; assert clazz != null : "No class matching id "+id; return clazz; } @Override public void compute2() { _clazz = TypeMap.className(_id); tryComplete(); }
public <TArg extends Arg> TArg extract(TArg arg) throws MissingArgumentError { Field[] fields = getFields(arg); int count = extract(arg, fields); if( arg instanceof Req && count != fields.length ) throw new MissingArgumentError(arg.usage()); return arg; }
public static void record_IOclose( AutoBuffer b, int flavor ) { H2ONode h2o = b._h2o==null ? H2O.SELF : b._h2o; // First long word going out has sender-port and a 'bad' control packet long b0 = UDP.udp.i_o.ordinal(); // Special flag to indicate io-record and not a rpc-record b0 |= H2O.SELF._key.udp_port()<<8; b0 |= flavor<<24; // I/O flavor; one of the Value.persist backends long iotime = b._time_start_ms > 0 ? (b._time_close_ms - b._time_start_ms) : 0; b0 |= iotime<<32; // msec from start-to-finish, including non-i/o overheads long b8 = b._size; // byte's transfered in this I/O long ns = b._time_io_ns; // nano's blocked doing I/O record2(h2o,ns,true,b.readMode()?1:0,0,b0,b8); }