From nextgens at freenetproject.org Tue Jul 1 03:15:15 2008 From: nextgens at freenetproject.org (nextgens at freenetproject.org) Date: Tue, 1 Jul 2008 03:15:15 +0000 (UTC) Subject: [freenet-cvs] r20883 - trunk/apps/new_installer/scripts Message-ID: <20080701031515.7858947997A@freenetproject.org> Author: nextgens Date: 2008-07-01 03:15:15 +0000 (Tue, 01 Jul 2008) New Revision: 20883 Modified: trunk/apps/new_installer/scripts/update.sh Log: update.sh: use openssl instead of sha1sum and md5sum as they aren't available on macos and *nix Modified: trunk/apps/new_installer/scripts/update.sh =================================================================== --- trunk/apps/new_installer/scripts/update.sh 2008-06-30 23:21:30 UTC (rev 20882) +++ trunk/apps/new_installer/scripts/update.sh 2008-07-01 03:15:15 UTC (rev 20883) @@ -3,7 +3,6 @@ CAFILE="startssl.pem" JOPTS="-Djava.net.preferIPv4Stack=true" SHA1_Sha1Test="5b69f30c827dc2e697ba043b075f1976a3fd9c2e" -MD5_Sha1Test="c46d4fb49ab86a8da3ff426e0933a63f" echo "Updating freenet" invert_return_code () { @@ -29,11 +28,11 @@ return 1 } -# Return the hash of a file (may be sha1 or md5) in the HASH variable +# Return the hash of a file in the HASH variable file_hash () { if test -n "$1" -a -e "$1" then - HASH="`$HASH_P \"$1\" | awk '{print $1;}'`" + HASH="`openssl md5 -sha1 \"$1\" | awk '{print $2;}'`" else HASH="NOT FOUND" fi @@ -53,18 +52,10 @@ fi } -# Determine which one we will use -if test ! -x "`which sha1sum`" +if test ! -x "`which openssl`" then - if test ! -x "`which md5sum`" - then - echo "No md5sum nor sha1sum utility detected; Please install one of those" - exit 1 - else - HASH_P="md5sum" - fi -else - HASH_P="sha1sum" + echo "No openssl utility detected; Please install it" + exit 1 fi # Attempt to use the auto-fetcher code, which will check the sha1sums. From nextgens at freenetproject.org Tue Jul 1 03:29:43 2008 From: nextgens at freenetproject.org (nextgens at freenetproject.org) Date: Tue, 1 Jul 2008 03:29:43 +0000 (UTC) Subject: [freenet-cvs] r20884 - trunk/apps/new_installer/scripts Message-ID: <20080701032943.3034F479759@freenetproject.org> Author: nextgens Date: 2008-07-01 03:29:42 +0000 (Tue, 01 Jul 2008) New Revision: 20884 Modified: trunk/apps/new_installer/scripts/update.sh Log: update.sh: paranoia Modified: trunk/apps/new_installer/scripts/update.sh =================================================================== --- trunk/apps/new_installer/scripts/update.sh 2008-07-01 03:15:15 UTC (rev 20883) +++ trunk/apps/new_installer/scripts/update.sh 2008-07-01 03:29:42 UTC (rev 20884) @@ -150,7 +150,6 @@ file_hash sha1test.jar case "$HASH" in $SHA1_Sha1Test) echo "The SHA1 of sha1test.jar matches";; - $MD5_Sha1Test) echo "The MD5 of sha1test.jar matches";; *) echo "sha1test.jar needs to be updated"; rm -f sha1test.jar;; esac From nextgens at freenetproject.org Tue Jul 1 04:07:26 2008 From: nextgens at freenetproject.org (nextgens at freenetproject.org) Date: Tue, 1 Jul 2008 04:07:26 +0000 (UTC) Subject: [freenet-cvs] r20885 - trunk/apps/new_installer/res/unix Message-ID: <20080701040726.71B25479998@freenetproject.org> Author: nextgens Date: 2008-07-01 04:07:26 +0000 (Tue, 01 Jul 2008) New Revision: 20885 Modified: trunk/apps/new_installer/res/unix/run.sh Log: new-installer: fix run.sh for macos tiger Modified: trunk/apps/new_installer/res/unix/run.sh =================================================================== --- trunk/apps/new_installer/res/unix/run.sh 2008-07-01 03:29:42 UTC (rev 20884) +++ trunk/apps/new_installer/res/unix/run.sh 2008-07-01 04:07:26 UTC (rev 20885) @@ -174,7 +174,7 @@ 'hp-ux' | 'hp-ux64') DIST_OS="hpux" ;; - 'darwin') + 'darwin' | 'oarwin') DIST_OS="macosx" #We use the 1.5 jvm if it exists From j16sdiz at freenetproject.org Tue Jul 1 07:36:21 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 07:36:21 +0000 (UTC) Subject: [freenet-cvs] r20886 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701073621.5291E4799D0@freenetproject.org> Author: j16sdiz Date: 2008-07-01 07:36:20 +0000 (Tue, 01 Jul 2008) New Revision: 20886 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: refactor to BatchProcessor interface and fix key count Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 04:07:26 UTC (rev 20885) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 07:36:20 UTC (rev 20886) @@ -11,7 +11,6 @@ import java.nio.channels.FileChannel; import java.security.MessageDigest; import java.text.DecimalFormat; -import java.util.AbstractList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; @@ -252,8 +251,10 @@ // Overwrite old offset Entry entry = new Entry(routingKey, header, data); - writeEntry(entry, oldOffset); // overwrite, don't increase keyCount + writeEntry(entry, oldOffset); writes.incrementAndGet(); + if (oldEntry.getGeneration() != generation) + keyCount.incrementAndGet(); return; } @@ -268,7 +269,7 @@ if (updateBloom) bloomFilter.updateFilter(getDigestedRoutingKey(routingKey)); writeEntry(entry, offset[i]); - long written = writes.incrementAndGet(); + writes.incrementAndGet(); keyCount.incrementAndGet(); return; @@ -280,8 +281,11 @@ Logger.debug(this, "collision, write to i=0, offset=" + offset[0]); if (updateBloom) bloomFilter.updateFilter(getDigestedRoutingKey(routingKey)); + oldEntry = readEntry(offset[0], null); writeEntry(entry, offset[0]); - long written = writes.incrementAndGet(); + writes.incrementAndGet(); + if (oldEntry.getGeneration() != generation) + keyCount.incrementAndGet(); } finally { unlockPlainKey(routingKey, false); } @@ -568,6 +572,14 @@ digestedRoutingKey = SaltedHashFreenetStore.this.getDigestedRoutingKey(this.plainRoutingKey); return digestedRoutingKey; } + + public byte getGeneration() { + return generation; + } + + public void setGeneration(byte generation) { + this.generation = generation; + } } /** @@ -820,7 +832,7 @@ /** * Write config file */ - private void writeConfigFile() throws IOException { + private void writeConfigFile() { configLock.writeLock().lock(); try { File tempConfig = new File(configFile.getPath() + ".tmp"); @@ -838,6 +850,8 @@ raf.close(); FileUtil.renameTo(tempConfig, configFile); + } catch (IOException ioe) { + Logger.error(this, "error writing config file for " + name, ioe); } finally { configLock.writeLock().unlock(); } @@ -849,6 +863,11 @@ private static Lock cleanerGlobalLock = new ReentrantLock(); // global across all datastore private Cleaner cleanerThread; + private interface BatchProcessor { + // return null to free the entry + Entry processs(Entry entry); + } + private class Cleaner extends Thread { /** * How often the clean should run @@ -898,12 +917,7 @@ } catch (Exception e) { // may throw IOException (even if it is not defined) Logger.error(this, "Can't force bloom filter", e); } - try { - writeConfigFile(); - } catch (IOException e) { - Logger.error(this, "Can't write config file", e); - } - + writeConfigFile(); cleanerLock.notifyAll(); try { @@ -930,7 +944,7 @@ initOldEntriesFile(); - List oldEntryList = new LinkedList(); + final List oldEntryList = new LinkedList(); // start from end of store, make store shrinking quicker long startOffset = (_prevStoreSize / RESIZE_MEMORY_ENTRIES) * RESIZE_MEMORY_ENTRIES; @@ -938,8 +952,16 @@ if (shutdown) return; - batchReadEntries(curOffset, RESIZE_MEMORY_ENTRIES, oldEntryList, true); + batchReadEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { + public Entry processs(Entry entry) { + if (entry.getStoreSize() == storeSize) // new size + return entry; + oldEntryList.add(entry); + return null; + } + }); + if (storeSize < _prevStoreSize) setStoreFileSize(Math.max(storeSize, curOffset)); @@ -984,32 +1006,33 @@ Logger.normal(this, "Start rebuilding bloom filter for " + callback); - bloomFilter.fork(); - List buildList = new AbstractList() { - @Override - public void add(int index, Entry entry) { - bloomFilter.updateFilter(entry.getDigestedRoutingKey()); - } - @Override - public Entry get(int index) { - return null; - } + configLock.writeLock().lock(); + try { + generation++; + bloomFilter.fork(); + keyCount.set(0); + } finally { + configLock.writeLock().unlock(); + } - @Override - public int size() { - return 0; - } - - }; - for (long curOffset = 0; curOffset < storeSize; curOffset += RESIZE_MEMORY_ENTRIES) { if (shutdown) { bloomFilter.discard(); return; } - batchReadEntries(curOffset, RESIZE_MEMORY_ENTRIES, buildList, false); + batchReadEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { + public Entry processs(Entry entry) { + if (entry.getGeneration() != generation) { + bloomFilter.updateFilter(entry.getDigestedRoutingKey()); + keyCount.incrementAndGet(); + } + return entry; + } + }); + Logger.normal(this, "Rebuilding bloom filter for " + callback + ": " + curOffset + "/" + storeSize); + writeConfigFile(); } bloomFilter.merge(); @@ -1024,23 +1047,19 @@ } /** - * Read a list of items from store. In resizing mode, only old items are read and the - * original offsets are freed. + * Read a list of items from store. * * @param offset * start offset, must be multiple of {@link FILE_SPLIT} * @param length * number of items to read, must be multiple of {@link FILE_SPLIT}. If this * excess store size, read as much as possible. - * @param items - * a list of items - * @param resizing - * If true, only get old items and free the offset. Otherwise, get - * all items. + * @param processor + * batch processor * @return true if operation complete successfully; false * otherwise (e.g. can't acquire locks, node shutting down) */ - private boolean batchReadEntries(long offset, int length, List items, boolean resizing) { + private boolean batchReadEntries(long offset, int length, BatchProcessor processor) { assert offset % FILE_SPLIT == 0; assert length % FILE_SPLIT == 0; @@ -1060,6 +1079,7 @@ ByteBuffer buf = ByteBuffer.allocate((int) bufLen); for (int i = 0; i < FILE_SPLIT; i++) { // for each split file + boolean dirty = false; buf.clear(); try { while (buf.hasRemaining()) { @@ -1074,27 +1094,31 @@ } buf.flip(); - for (int j = 0; buf.remaining() >= entryTotalLength; j++) { - if (shutdown) - return false; + try { + for (int j = 0; buf.limit() >= j * entryTotalLength; j++) { + if (shutdown) + return false; - ByteBuffer enBuf = buf.slice(); - buf.position(buf.position() + (int) entryTotalLength); + buf.position((int) (j * entryTotalLength)); + if (buf.remaining() < entryTotalLength) // EOF + break; - enBuf.limit((int) entryTotalLength); + ByteBuffer enBuf = buf.slice(); + enBuf.limit((int) entryTotalLength); - Entry entry = new Entry(enBuf); - entry.curOffset = offset + j * FILE_SPLIT + i; + Entry entry = new Entry(enBuf); + entry.curOffset = offset + j * FILE_SPLIT + i; - try { if (entry.isFree()) continue; // not occupied - if (resizing && entry.storeSize != storeSize) - continue; // resizing mode, not new item - items.add(entry); - - if (resizing) { // free the offset + Entry newEntry = processor.processs(entry); + if (newEntry != null) { + // write back + buf.position((int) (j * entryTotalLength)); + buf.put(newEntry.toByteBuffer()); + dirty = true; + } else { // free the offset try { freeOffset(entry.curOffset); keyCount.decrementAndGet(); @@ -1103,11 +1127,20 @@ Logger.error(this, "error freeing entry " + entry.curOffset, ioe); } } - } finally { - // unlock current entry - unlockEntry(entry.curOffset); - locked[(int) (entry.curOffset - offset)] = false; } + } finally { + // write back. + if (dirty) { + buf.flip(); + + try { + while (buf.hasRemaining()) { + storeFC[i].write(buf, startFileOffset + buf.position()); + } + } catch (IOException ioe) { + Logger.error(this, "unexpected IOException", ioe); + } + } } } @@ -1401,11 +1434,7 @@ try { flushAndClose(); flags &= ~FLAG_DIRTY; // clean shutdown - try { - writeConfigFile(); - } catch (IOException e) { - Logger.error(this, "error writing store config", e); - } + writeConfigFile(); } finally { configLock.writeLock().unlock(); } From j16sdiz at freenetproject.org Tue Jul 1 08:09:13 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 08:09:13 +0000 (UTC) Subject: [freenet-cvs] r20887 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701080913.8958A47972F@freenetproject.org> Author: j16sdiz Date: 2008-07-01 08:09:13 +0000 (Tue, 01 Jul 2008) New Revision: 20887 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: don't write not-modified entry Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 07:36:20 UTC (rev 20886) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 08:09:13 UTC (rev 20887) @@ -434,6 +434,9 @@ isEncrypted = true; } + private Entry() { + } + public ByteBuffer toByteBuffer() { ByteBuffer out = ByteBuffer.allocate((int) entryTotalLength); encrypt(); @@ -863,8 +866,11 @@ private static Lock cleanerGlobalLock = new ReentrantLock(); // global across all datastore private Cleaner cleanerThread; - private interface BatchProcessor { + private final Entry NOT_MODIFIED = new Entry(); + + private interface BatchProcessor { // return null to free the entry + // return NOT_MODIFIED to keep the old entry Entry processs(Entry entry); } @@ -955,7 +961,7 @@ batchReadEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { public Entry processs(Entry entry) { if (entry.getStoreSize() == storeSize) // new size - return entry; + return NOT_MODIFIED; oldEntryList.add(entry); return null; @@ -1026,8 +1032,11 @@ if (entry.getGeneration() != generation) { bloomFilter.updateFilter(entry.getDigestedRoutingKey()); keyCount.incrementAndGet(); + + entry.setGeneration(generation); + return entry; } - return entry; + return NOT_MODIFIED; } }); @@ -1113,12 +1122,7 @@ continue; // not occupied Entry newEntry = processor.processs(entry); - if (newEntry != null) { - // write back - buf.position((int) (j * entryTotalLength)); - buf.put(newEntry.toByteBuffer()); - dirty = true; - } else { // free the offset + if (newEntry == null) {// free the offset try { freeOffset(entry.curOffset); keyCount.decrementAndGet(); @@ -1126,7 +1130,13 @@ if (!shutdown) Logger.error(this, "error freeing entry " + entry.curOffset, ioe); } - } + } else if (newEntry == NOT_MODIFIED) { + } else { + // write back + buf.position((int) (j * entryTotalLength)); + buf.put(newEntry.toByteBuffer()); + dirty = true; + } } } finally { // write back. From j16sdiz at freenetproject.org Tue Jul 1 08:30:31 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 08:30:31 +0000 (UTC) Subject: [freenet-cvs] r20888 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701083031.907844797BE@freenetproject.org> Author: j16sdiz Date: 2008-07-01 08:30:31 +0000 (Tue, 01 Jul 2008) New Revision: 20888 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: rename batchReadEntries to batchProcessEntries Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 08:09:13 UTC (rev 20887) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 08:30:31 UTC (rev 20888) @@ -958,7 +958,7 @@ if (shutdown) return; - batchReadEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { + batchProcessEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { public Entry processs(Entry entry) { if (entry.getStoreSize() == storeSize) // new size return NOT_MODIFIED; @@ -1027,7 +1027,7 @@ bloomFilter.discard(); return; } - batchReadEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { + batchProcessEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { public Entry processs(Entry entry) { if (entry.getGeneration() != generation) { bloomFilter.updateFilter(entry.getDigestedRoutingKey()); @@ -1068,7 +1068,7 @@ * @return true if operation complete successfully; false * otherwise (e.g. can't acquire locks, node shutting down) */ - private boolean batchReadEntries(long offset, int length, BatchProcessor processor) { + private boolean batchProcessEntries(long offset, int length, BatchProcessor processor) { assert offset % FILE_SPLIT == 0; assert length % FILE_SPLIT == 0; From j16sdiz at freenetproject.org Tue Jul 1 08:30:55 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 08:30:55 +0000 (UTC) Subject: [freenet-cvs] r20889 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701083055.E969447972F@freenetproject.org> Author: j16sdiz Date: 2008-07-01 08:30:55 +0000 (Tue, 01 Jul 2008) New Revision: 20889 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: log Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 08:30:31 UTC (rev 20888) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 08:30:55 UTC (rev 20889) @@ -220,7 +220,7 @@ public void put(StorableBlock block, byte[] routingKey, byte[] fullKey, byte[] data, byte[] header, boolean overwrite) throws IOException, KeyCollisionException { if (logMINOR) - Logger.minor(this, "Putting " + HexUtil.bytesToHex(routingKey) + " for " + callback); + Logger.minor(this, "Putting " + HexUtil.bytesToHex(routingKey) + " (" + name + ")"); configLock.readLock().lock(); try { @@ -986,13 +986,13 @@ } long processed = _prevStoreSize - curOffset; - Logger.normal(this, "Store resize " + callback + ": " + processed + "/" + _prevStoreSize); + Logger.normal(this, "Store resize (" + name + "): " + processed + "/" + _prevStoreSize); } resolveOldEntriesFile(); long endTime = System.currentTimeMillis(); - Logger.normal(this, "Finish resizing " + callback + " in " + (endTime - startTime) / 1000 + "s"); + Logger.normal(this, "Finish resizing (" + name + ") in " + (endTime - startTime) / 1000 + "s"); configLock.writeLock().lock(); try { @@ -1010,9 +1010,8 @@ if (bloomFilter == null) return; - Logger.normal(this, "Start rebuilding bloom filter for " + callback); + Logger.normal(this, "Start rebuilding bloom filter (" + name + ")"); - configLock.writeLock().lock(); try { generation++; @@ -1040,12 +1039,12 @@ } }); - Logger.normal(this, "Rebuilding bloom filter for " + callback + ": " + curOffset + "/" + storeSize); + Logger.normal(this, "Rebuilding bloom filter (" + name + "): " + curOffset + "/" + storeSize); writeConfigFile(); } bloomFilter.merge(); - Logger.normal(this, "Finish rebuilding bloom filter for " + callback); + Logger.normal(this, "Finish rebuilding bloom filter (" + name + ")"); configLock.writeLock().lock(); try { From j16sdiz at freenetproject.org Tue Jul 1 09:27:11 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 09:27:11 +0000 (UTC) Subject: [freenet-cvs] r20890 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701092711.1ACAE479986@freenetproject.org> Author: j16sdiz Date: 2008-07-01 09:27:10 +0000 (Tue, 01 Jul 2008) New Revision: 20890 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: don't update bloom twice Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 08:30:55 UTC (rev 20889) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:27:10 UTC (rev 20890) @@ -157,8 +157,6 @@ try { StorableBlock block = entry.getStorableBlock(routingKey, fullKey); hits.incrementAndGet(); - if (updateBloom && !checkBloom) - bloomFilter.updateFilter(getDigestedRoutingKey(routingKey)); return block; } catch (KeyVerifyException e) { Logger.minor(this, "key verification exception", e); From j16sdiz at freenetproject.org Tue Jul 1 09:27:35 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 09:27:35 +0000 (UTC) Subject: [freenet-cvs] r20891 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701092735.A599847972F@freenetproject.org> Author: j16sdiz Date: 2008-07-01 09:27:35 +0000 (Tue, 01 Jul 2008) New Revision: 20891 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: log Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:27:10 UTC (rev 20890) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:27:35 UTC (rev 20891) @@ -952,6 +952,7 @@ // start from end of store, make store shrinking quicker long startOffset = (_prevStoreSize / RESIZE_MEMORY_ENTRIES) * RESIZE_MEMORY_ENTRIES; + int i = 0; for (long curOffset = startOffset; curOffset >= 0; curOffset -= RESIZE_MEMORY_ENTRIES) { if (shutdown) return; @@ -984,7 +985,8 @@ } long processed = _prevStoreSize - curOffset; - Logger.normal(this, "Store resize (" + name + "): " + processed + "/" + _prevStoreSize); + if (i++ % 16 == 0) + Logger.normal(this, "Store resize (" + name + "): " + processed + "/" + _prevStoreSize); } resolveOldEntriesFile(); @@ -1009,6 +1011,7 @@ return; Logger.normal(this, "Start rebuilding bloom filter (" + name + ")"); + long startTime = System.currentTimeMillis(); configLock.writeLock().lock(); try { @@ -1019,6 +1022,7 @@ configLock.writeLock().unlock(); } + int i = 0; for (long curOffset = 0; curOffset < storeSize; curOffset += RESIZE_MEMORY_ENTRIES) { if (shutdown) { bloomFilter.discard(); @@ -1037,12 +1041,16 @@ } }); - Logger.normal(this, "Rebuilding bloom filter (" + name + "): " + curOffset + "/" + storeSize); - writeConfigFile(); + if (i++ % 16 == 0) { + Logger.normal(this, "Rebuilding bloom filter (" + name + "): " + curOffset + "/" + storeSize); + writeConfigFile(); + } } bloomFilter.merge(); - Logger.normal(this, "Finish rebuilding bloom filter (" + name + ")"); + long endTime = System.currentTimeMillis(); + Logger.normal(this, "Finish rebuilding bloom filter (" + name + ") in " + (endTime - startTime) / 1000 + + "s"); configLock.writeLock().lock(); try { From j16sdiz at freenetproject.org Tue Jul 1 09:27:58 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 09:27:58 +0000 (UTC) Subject: [freenet-cvs] r20892 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701092758.E0D2847972F@freenetproject.org> Author: j16sdiz Date: 2008-07-01 09:27:58 +0000 (Tue, 01 Jul 2008) New Revision: 20892 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: prevent http timeout on store resize Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:27:35 UTC (rev 20891) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:27:58 UTC (rev 20892) @@ -860,7 +860,8 @@ // ------------- Store resizing private long prevStoreSize = 0; - private Object cleanerLock = new Object(); // local to this datastore + private Lock cleanerLock = new ReentrantLock(); // local to this datastore + private Condition cleanerCondition = cleanerLock.newCondition(); private static Lock cleanerGlobalLock = new ReentrantLock(); // global across all datastore private Cleaner cleanerThread; @@ -887,7 +888,8 @@ @Override public void run() { while (!shutdown) { - synchronized (cleanerLock) { + cleanerLock.lock(); + try { long _prevStoreSize; boolean _rebuildBloom; @@ -907,7 +909,7 @@ } } - if (_rebuildBloom && cleanerGlobalLock.tryLock()) { + if (_rebuildBloom && prevStoreSize == 0 && cleanerGlobalLock.tryLock()) { try { rebuildBloom(); } finally { @@ -922,13 +924,14 @@ Logger.error(this, "Can't force bloom filter", e); } writeConfigFile(); - cleanerLock.notifyAll(); try { - cleanerLock.wait(CLEANER_PERIOD); + cleanerCondition.await(CLEANER_PERIOD, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { Logger.debug(this, "interrupted", e); } + } finally { + cleanerLock.unlock(); } } } @@ -954,7 +957,7 @@ long startOffset = (_prevStoreSize / RESIZE_MEMORY_ENTRIES) * RESIZE_MEMORY_ENTRIES; int i = 0; for (long curOffset = startOffset; curOffset >= 0; curOffset -= RESIZE_MEMORY_ENTRIES) { - if (shutdown) + if (shutdown || _prevStoreSize != prevStoreSize) return; batchProcessEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { @@ -996,7 +999,8 @@ configLock.writeLock().lock(); try { - assert _prevStoreSize == prevStoreSize; + if (_prevStoreSize != prevStoreSize) + return; prevStoreSize = 0; } finally { configLock.writeLock().unlock(); @@ -1024,7 +1028,7 @@ int i = 0; for (long curOffset = 0; curOffset < storeSize; curOffset += RESIZE_MEMORY_ENTRIES) { - if (shutdown) { + if (shutdown || prevStoreSize != 0) { bloomFilter.discard(); return; } @@ -1283,27 +1287,28 @@ public void setMaxKeys(long newStoreSize, boolean shrinkNow) throws IOException { Logger.normal(this, "[" + name + "] Resize newStoreSize=" + newStoreSize + ", shinkNow=" + shrinkNow); - synchronized (cleanerLock) { - configLock.writeLock().lock(); - try { - if (newStoreSize == this.storeSize) - return; + configLock.writeLock().lock(); + try { + if (newStoreSize == this.storeSize) + return; - if (prevStoreSize != 0) { - Logger.normal(this, "[" + name + "] resize already in progress, ignore resize request"); - return; - } - - prevStoreSize = storeSize; - storeSize = newStoreSize; - flags |= FLAG_REBUILD_BLOOM; - writeConfigFile(); - } finally { - configLock.writeLock().unlock(); + if (prevStoreSize != 0) { + Logger.normal(this, "[" + name + "] resize already in progress, ignore resize request"); + return; } - cleanerLock.notify(); + prevStoreSize = storeSize; + storeSize = newStoreSize; + flags |= FLAG_REBUILD_BLOOM; + writeConfigFile(); + } finally { + configLock.writeLock().unlock(); } + + if (cleanerLock.tryLock()) { + cleanerCondition.signal(); + cleanerLock.unlock(); + } } // ------------- Locking From j16sdiz at freenetproject.org Tue Jul 1 09:45:43 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 09:45:43 +0000 (UTC) Subject: [freenet-cvs] r20893 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701094543.B2A594793D6@freenetproject.org> Author: j16sdiz Date: 2008-07-01 09:45:43 +0000 (Tue, 01 Jul 2008) New Revision: 20893 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: force resize on restart Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:27:58 UTC (rev 20892) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:45:43 UTC (rev 20893) @@ -126,10 +126,18 @@ flags |= FLAG_DIRTY; // datastore is now dirty until flushAndClose() writeConfigFile(); + + if (maxKeys != storeSize) { + if (prevStoreSize != 0) { + storeSize = prevStoreSize; + prevStoreSize = 0; + } + setMaxKeys(maxKeys, true); + } callback.setStore(this); shutdownHook.addEarlyJob(new Thread(new ShutdownDB())); - + cleanerThread = new Cleaner(); cleanerThread.start(); } From j16sdiz at freenetproject.org Tue Jul 1 09:46:08 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 09:46:08 +0000 (UTC) Subject: [freenet-cvs] r20894 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701094608.2B28D4793D6@freenetproject.org> Author: j16sdiz Date: 2008-07-01 09:46:07 +0000 (Tue, 01 Jul 2008) New Revision: 20894 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: no duplicated effort Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:45:43 UTC (rev 20893) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:46:07 UTC (rev 20894) @@ -830,7 +830,7 @@ generation = raf.readByte(); flags = raf.readByte(); - if (prevStoreSize != 0 || (flags & FLAG_DIRTY) != 0) + if ((flags & FLAG_DIRTY) != 0) flags |= FLAG_REBUILD_BLOOM; raf.close(); From j16sdiz at freenetproject.org Tue Jul 1 10:29:06 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 10:29:06 +0000 (UTC) Subject: [freenet-cvs] r20895 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701102906.5CA6547995B@freenetproject.org> Author: j16sdiz Date: 2008-07-01 10:29:05 +0000 (Tue, 01 Jul 2008) New Revision: 20895 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: digested key cache Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 09:46:07 UTC (rev 20894) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 10:29:05 UTC (rev 20895) @@ -13,6 +13,7 @@ import java.text.DecimalFormat; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.ListIterator; @@ -39,6 +40,7 @@ import freenet.keys.KeyVerifyException; import freenet.node.SemiOrderedShutdownHook; import freenet.support.BloomFilter; +import freenet.support.ByteArrayWrapper; import freenet.support.Fields; import freenet.support.HexUtil; import freenet.support.Logger; @@ -1475,14 +1477,27 @@ */ private byte[] salt; + private Map digestRoutingKeyCache = new LinkedHashMap() { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > 128; + } + }; + /** * Get hashed routing key * * @param routingKey * @return */ - // TODO use a little cache? private byte[] getDigestedRoutingKey(byte[] routingKey) { + ByteArrayWrapper key = new ByteArrayWrapper(routingKey); + synchronized (digestRoutingKeyCache) { + byte[] dk = digestRoutingKeyCache.get(key); + if (dk != null) + return dk; + } + MessageDigest digest = SHA256.getMessageDigest(); try { digest.update(routingKey); @@ -1491,6 +1506,10 @@ byte[] hashedRoutingKey = digest.digest(); assert hashedRoutingKey.length == 0x20; + synchronized (digestRoutingKeyCache) { + digestRoutingKeyCache.put(key, hashedRoutingKey); + } + return hashedRoutingKey; } finally { SHA256.returnMessageDigest(digest); From toad at freenetproject.org Tue Jul 1 10:41:01 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 10:41:01 +0000 (UTC) Subject: [freenet-cvs] r20896 - branches/db4o/freenet/src/freenet/node/fcp Message-ID: <20080701104101.681A747994E@freenetproject.org> Author: toad Date: 2008-07-01 10:41:01 +0000 (Tue, 01 Jul 2008) New Revision: 20896 Modified: branches/db4o/freenet/src/freenet/node/fcp/FCPClient.java Log: Fix ClassCastException. I don't think this code exists on trunk, iirc we handle it differently there. Modified: branches/db4o/freenet/src/freenet/node/fcp/FCPClient.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/FCPClient.java 2008-07-01 10:29:05 UTC (rev 20895) +++ branches/db4o/freenet/src/freenet/node/fcp/FCPClient.java 2008-07-01 10:41:01 UTC (rev 20896) @@ -403,8 +403,12 @@ public ClientGet getCompletedRequest(FreenetURI key, ObjectContainer container) { // FIXME speed this up with another hashmap or something. + // FIXME keep a transient hashmap in RAM, use it for fproxy. + // FIXME consider supporting inserts too. for(int i=0;i Author: toad Date: 2008-07-01 10:59:34 +0000 (Tue, 01 Jul 2008) New Revision: 20897 Modified: branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java branches/db4o/freenet/src/freenet/support/RandomGrabArray.java Log: Only reschedule on coming out of cooldown if not already scheduled. If appear to be scheduled, check with the RGA whether we really are, complain if not. Modified: branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java 2008-07-01 10:41:01 UTC (rev 20896) +++ branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java 2008-07-01 10:59:34 UTC (rev 20897) @@ -30,6 +30,7 @@ import freenet.keys.NodeCHK; import freenet.node.RequestScheduler; import freenet.support.Logger; +import freenet.support.RandomGrabArray; import freenet.support.api.Bucket; import freenet.support.io.BucketTools; @@ -807,7 +808,20 @@ if(v != null) { for(int i=0;i Author: toad Date: 2008-07-01 11:02:12 +0000 (Tue, 01 Jul 2008) New Revision: 20898 Modified: branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java Log: Fix NPE Modified: branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java 2008-07-01 10:59:34 UTC (rev 20897) +++ branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java 2008-07-01 11:02:12 UTC (rev 20898) @@ -191,6 +191,8 @@ if(Logger.shouldLog(Logger.MINOR, this)) Logger.minor(this, "Not requeueing as deadline has not passed yet"); return; } + if(persistent) + container.activate(this.key, 5); if(!(key.equals(this.key.getNodeKey()))) { Logger.error(this, "Got requeueAfterCooldown for wrong key: "+key+" but mine is "+this.key.getNodeKey()+" for "+this.key); return; From j16sdiz at freenetproject.org Tue Jul 1 11:05:55 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 11:05:55 +0000 (UTC) Subject: [freenet-cvs] r20899 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701110555.3C8FE479948@freenetproject.org> Author: j16sdiz Date: 2008-07-01 11:05:55 +0000 (Tue, 01 Jul 2008) New Revision: 20899 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: batch free Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 11:02:12 UTC (rev 20898) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 11:05:55 UTC (rev 20899) @@ -1142,13 +1142,11 @@ Entry newEntry = processor.processs(entry); if (newEntry == null) {// free the offset - try { - freeOffset(entry.curOffset); - keyCount.decrementAndGet(); - } catch (IOException ioe) { - if (!shutdown) - Logger.error(this, "error freeing entry " + entry.curOffset, ioe); - } + buf.position((int) (j * entryTotalLength)); + buf.put(ByteBuffer.allocate((int) entryTotalLength)); + keyCount.decrementAndGet(); + + dirty = true; } else if (newEntry == NOT_MODIFIED) { } else { // write back From j16sdiz at freenetproject.org Tue Jul 1 11:06:17 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 11:06:17 +0000 (UTC) Subject: [freenet-cvs] r20900 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701110617.6002B47995B@freenetproject.org> Author: j16sdiz Date: 2008-07-01 11:06:17 +0000 (Tue, 01 Jul 2008) New Revision: 20900 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: typo Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 11:05:55 UTC (rev 20899) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 11:06:17 UTC (rev 20900) @@ -880,7 +880,7 @@ private interface BatchProcessor { // return null to free the entry // return NOT_MODIFIED to keep the old entry - Entry processs(Entry entry); + Entry process(Entry entry); } private class Cleaner extends Thread { @@ -971,7 +971,7 @@ return; batchProcessEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { - public Entry processs(Entry entry) { + public Entry process(Entry entry) { if (entry.getStoreSize() == storeSize) // new size return NOT_MODIFIED; @@ -1043,7 +1043,7 @@ return; } batchProcessEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { - public Entry processs(Entry entry) { + public Entry process(Entry entry) { if (entry.getGeneration() != generation) { bloomFilter.updateFilter(entry.getDigestedRoutingKey()); keyCount.incrementAndGet(); @@ -1140,7 +1140,7 @@ if (entry.isFree()) continue; // not occupied - Entry newEntry = processor.processs(entry); + Entry newEntry = processor.process(entry); if (newEntry == null) {// free the offset buf.position((int) (j * entryTotalLength)); buf.put(ByteBuffer.allocate((int) entryTotalLength)); From toad at freenetproject.org Tue Jul 1 11:24:59 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 11:24:59 +0000 (UTC) Subject: [freenet-cvs] r20901 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701112459.9C2CC47995B@freenetproject.org> Author: toad Date: 2008-07-01 11:24:59 +0000 (Tue, 01 Jul 2008) New Revision: 20901 Modified: branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java Log: Fix spurious "Not in archive" Modified: branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java 2008-07-01 11:06:17 UTC (rev 20900) +++ branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java 2008-07-01 11:24:59 UTC (rev 20901) @@ -280,16 +280,23 @@ throw new FetchException(FetchException.NOT_ENOUGH_PATH_COMPONENTS, -1, false, null, uri.addMetaStrings(new String[] { "" })); else name = removeMetaString(); // Since metadata is a document, we just replace metadata here - if(logMINOR) Logger.minor(this, "Next meta-string: "+name); + if(logMINOR) Logger.minor(this, "Next meta-string: "+name+" length "+name.length()+" for "+this); if(name == null) { metadata = metadata.getDefaultDocument(); - if(persistent) container.set(this); + if(persistent) { + container.set(this); + container.set(metaStrings); + } if(metadata == null) throw new FetchException(FetchException.NOT_ENOUGH_PATH_COMPONENTS, -1, false, null, uri.addMetaStrings(new String[] { "" })); } else { metadata = metadata.getDocument(name); thisKey = thisKey.pushMetaString(name); - if(persistent) container.set(this); + if(persistent) { + container.set(this); + container.set(metaStrings); + container.set(thisKey); + } if(metadata == null) throw new FetchException(FetchException.NOT_IN_ARCHIVE); } From toad at freenetproject.org Tue Jul 1 12:16:03 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 12:16:03 +0000 (UTC) Subject: [freenet-cvs] r20902 - in branches/db4o/freenet/src/freenet: client/async node Message-ID: <20080701121603.A90D4479986@freenetproject.org> Author: toad Date: 2008-07-01 12:16:03 +0000 (Tue, 01 Jul 2008) New Revision: 20902 Modified: branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java branches/db4o/freenet/src/freenet/client/async/ClientGetState.java branches/db4o/freenet/src/freenet/client/async/ClientGetter.java branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java branches/db4o/freenet/src/freenet/client/async/SingleBlockInserter.java branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSegment.java branches/db4o/freenet/src/freenet/client/async/SplitFileFetcherSubSegment.java branches/db4o/freenet/src/freenet/client/async/USKFetcher.java branches/db4o/freenet/src/freenet/client/async/USKFetcherTag.java branches/db4o/freenet/src/freenet/client/async/USKInserter.java branches/db4o/freenet/src/freenet/client/async/USKManager.java branches/db4o/freenet/src/freenet/client/async/USKRetriever.java branches/db4o/freenet/src/freenet/node/SendableGet.java branches/db4o/freenet/src/freenet/node/SimpleSendableInsert.java Log: When registering a big (multi-segment) splitfile, add the RegisterMe and schedule each segment as a separate job. On a large splitfile (ISO size), we were getting *very* long stalls while every block in the splitfile was addPendingKeys()'ed. Modified: branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java 2008-07-01 11:24:59 UTC (rev 20901) +++ branches/db4o/freenet/src/freenet/client/async/BaseSingleFileFetcher.java 2008-07-01 12:16:03 UTC (rev 20902) @@ -95,7 +95,7 @@ } return true; // We will retry, just not yet. See requeueAfterCooldown(Key). } else { - schedule(container, context, true); + schedule(container, context, false, true); } return true; } @@ -199,7 +199,7 @@ } if(Logger.shouldLog(Logger.MINOR, this)) Logger.minor(this, "Requeueing after cooldown "+key+" for "+this); - schedule(container, context, true); + schedule(container, context, false, true); } } Modified: branches/db4o/freenet/src/freenet/client/async/ClientGetState.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientGetState.java 2008-07-01 11:24:59 UTC (rev 20901) +++ branches/db4o/freenet/src/freenet/client/async/ClientGetState.java 2008-07-01 12:16:03 UTC (rev 20902) @@ -11,7 +11,7 @@ */ public interface ClientGetState { - public void schedule(ObjectContainer container, ClientContext context, boolean probablyNotInStore); + public void schedule(ObjectContainer container, ClientContext context, boolean delayedRegister, boolean probablyNotInStore); public void cancel(ObjectContainer container, ClientContext context); Modified: branches/db4o/freenet/src/freenet/client/async/ClientGetter.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientGetter.java 2008-07-01 11:24:59 UTC (rev 20901) +++ branches/db4o/freenet/src/freenet/client/async/ClientGetter.java 2008-07-01 12:16:03 UTC (rev 20902) @@ -113,7 +113,7 @@ return false; } } - currentState.schedule(container, context, false); + currentState.schedule(container, context, false, false); } if(cancelled) cancel(); } catch (MalformedURLException e) { Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 11:24:59 UTC (rev 20901) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 12:16:03 UTC (rev 20902) @@ -155,14 +155,14 @@ choosenPriorityScheduler = val; } - public void register(final SendableRequest req, boolean probablyNotInStore) { - register(req, databaseExecutor.onThread(), null, probablyNotInStore); + public void register(final SendableRequest req, boolean regmeOnly, boolean probablyNotInStore) { + register(req, databaseExecutor.onThread(), regmeOnly, null, probablyNotInStore); } /** * Register and then delete the RegisterMe which is passed in to avoid querying. */ - public void register(final SendableRequest req, boolean onDatabaseThread, RegisterMe reg, final boolean probablyNotInStore) { + public void register(final SendableRequest req, boolean onDatabaseThread, final boolean regmeOnly, RegisterMe reg, final boolean probablyNotInStore) { logMINOR = Logger.shouldLog(Logger.MINOR, this); if(logMINOR) Logger.minor(this, "Registering "+req, new Exception("debug")); final boolean persistent = req.persistent(); @@ -172,6 +172,19 @@ final SendableGet getter = (SendableGet)req; if(persistent && onDatabaseThread) { + if(regmeOnly) { + assert(reg == null); + reg = schedCore.queueRegister(getter, databaseExecutor, selectorContainer); + final RegisterMe regme = reg; + clientContext.jobRunner.queue(new DBJob() { + + public void run(ObjectContainer container, ClientContext context) { + register(req, true, false, regme, probablyNotInStore); + } + + }, NativeThread.NORM_PRIORITY+1, false); + return; + } schedCore.addPendingKeys(getter, selectorContainer); final Object[] keyTokens = getter.sendableKeys(selectorContainer); final ClientKey[] keys = new ClientKey[keyTokens.length]; Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 11:24:59 UTC (rev 20901) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 12:16:03 UTC (rev 20902) @@ -530,7 +530,7 @@ Logger.minor(this, "Running RegisterMe for "+reg.getter+" : "+reg.key.addedTime+" : "+reg.key.priority); // Don't need to activate, fields should exist? FIXME try { - sched.register(reg.getter, true, reg, false); + sched.register(reg.getter, true, false, reg, false); } catch (Throwable t) { Logger.error(this, "Caught "+t+" running RegisterMeRunner", t); // Cancel the request, and commit so it isn't tried again. Modified: branches/db4o/freenet/src/freenet/client/async/SingleBlockInserter.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/SingleBlockInserter.java 2008-07-01 11:24:59 UTC (rev 20901) +++ branches/db4o/freenet/src/freenet/client/async/SingleBlockInserter.java 2008-07-01 12:16:03 UTC (rev 20902) @@ -199,7 +199,7 @@ } if(persistent) container.set(this); - getScheduler(context).register(this, false); + getScheduler(context).register(this, false, false); } private void fail(InsertException e, ObjectContainer container, ClientContext context) { @@ -263,7 +263,7 @@ if(persistent) container.set(this); } else { - getScheduler(context).register(this, false); + getScheduler(context).register(this, false, false); } } Modified: branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java 2008-07-01 11:24:59 UTC (rev 20901) +++ branches/db4o/freenet/src/freenet/client/async/SingleFileFetcher.java 2008-07-01 12:16:03 UTC (rev 20902) @@ -522,7 +522,7 @@ f.addDecompressor(codec); } parent.onTransition(this, f, container); - f.schedule(container, context, false); + f.schedule(container, context, false, false); if(persistent) { container.set(metaStrings); container.set(this); @@ -599,7 +599,7 @@ SplitFileFetcher sf = new SplitFileFetcher(metadata, rcb, parent, ctx, decompressors, clientMetadata, actx, recursionLevel, returnBucket, token, container); parent.onTransition(this, sf, container); - sf.schedule(container, context, false); + sf.schedule(container, context, false, false); rcb.onBlockSetFinished(this, container, context); // Clear our own metadata, we won't need it any more. // For multi-level metadata etc see above. @@ -925,7 +925,7 @@ if(l == usk.suggestedEdition) { SingleFileFetcher sf = new SingleFileFetcher(parent, cb, clientMetadata, key, metaStrings, key.getURI().addMetaStrings(metaStrings), 0, ctx, actx, null, null, maxRetries, recursionLevel+1, dontTellClientGet, token, false, returnBucket, true, container, context); - sf.schedule(container, context, false); + sf.schedule(container, context, false, false); } else { cb.onFailure(new FetchException(FetchException.PERMANENT_REDIRECT, newUSK.getURI().addMetaStrings(metaStrings)), null, container, context); } Modified: branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java 2008-07-01 11:24:59 UTC (rev 20901) +++ branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java 2008-07-01 12:16:03 UTC (rev 20902) @@ -326,12 +326,14 @@ } } - public void schedule(ObjectContainer container, ClientContext context, boolean probablyNotInStore) { + public void schedule(ObjectContainer container, ClientContext context, boolean regmeOnly, boolean probablyNotInStore) { if(persistent) container.activate(this, 1); + if(segments.length > 1) + regmeOnly = true; if(Logger.shouldLog(Logger.MINOR, this)) Logger.minor(this, "Scheduling "+this); for(int i=0;i Author: toad Date: 2008-07-01 12:27:58 +0000 (Tue, 01 Jul 2008) New Revision: 20903 Modified: branches/db4o/freenet/src/freenet/keys/FreenetURI.java Log: Logging Modified: branches/db4o/freenet/src/freenet/keys/FreenetURI.java =================================================================== --- branches/db4o/freenet/src/freenet/keys/FreenetURI.java 2008-07-01 12:16:03 UTC (rev 20902) +++ branches/db4o/freenet/src/freenet/keys/FreenetURI.java 2008-07-01 12:27:58 UTC (rev 20903) @@ -544,6 +544,8 @@ public String toString(boolean prefix, boolean pureAscii) { if(keyType == null) { // Not activated or something... + if(Logger.shouldLog(Logger.MINOR, this)) + Logger.minor(this, "Not activated?? in toString("+prefix+","+pureAscii+")"); return null; } StringBuffer b; From toad at freenetproject.org Tue Jul 1 12:52:58 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 12:52:58 +0000 (UTC) Subject: [freenet-cvs] r20904 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701125258.7E6DE479A00@freenetproject.org> Author: toad Date: 2008-07-01 12:52:58 +0000 (Tue, 01 Jul 2008) New Revision: 20904 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java Log: Tweak priorities Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 12:27:58 UTC (rev 20903) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 12:52:58 UTC (rev 20904) @@ -181,8 +181,8 @@ public void run(ObjectContainer container, ClientContext context) { register(req, true, false, regme, probablyNotInStore); } - - }, NativeThread.NORM_PRIORITY+1, false); + // NORM_PRIORITY so the completion (finishRegister()) runs before the next block does addPendingKeys(). + }, NativeThread.NORM_PRIORITY, false); return; } schedCore.addPendingKeys(getter, selectorContainer); @@ -346,8 +346,8 @@ container.activate(g, 1); g.onFailure(new LowLevelGetException(LowLevelGetException.DECODE_FAILED), token, container, context); } - - }, NativeThread.NORM_PRIORITY, false); + // NORM_PRIORITY+1 as must run before finishRegister() + }, NativeThread.NORM_PRIORITY+1, false); } continue; // other keys might be valid } @@ -366,8 +366,8 @@ container.activate(g, 1); g.onSuccess(b, true, t, container, context); } - - }, NativeThread.NORM_PRIORITY, false); + // NORM_PRIORITY+1 as must run before finishRegister() + }, NativeThread.NORM_PRIORITY+1, false); } else { g.onSuccess(b, true, t, null, clientContext); } @@ -411,7 +411,7 @@ starter.wakeUp(); } - }, NativeThread.NORM_PRIORITY, false); + }, NativeThread.NORM_PRIORITY+1, false); } } else { // Register immediately. From toad at freenetproject.org Tue Jul 1 12:58:33 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 12:58:33 +0000 (UTC) Subject: [freenet-cvs] r20905 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701125834.0A18F479A45@freenetproject.org> Author: toad Date: 2008-07-01 12:58:33 +0000 (Tue, 01 Jul 2008) New Revision: 20905 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java Log: Logging Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 12:52:58 UTC (rev 20904) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 12:58:33 UTC (rev 20905) @@ -503,9 +503,6 @@ private DBJob requestStarterQueueFiller = new DBJob() { public void run(ObjectContainer container, ClientContext context) { - if(isInsertScheduler && !isSSKScheduler) { - if(logMINOR) Logger.minor(this, "Scheduling inserts..."); - } if(logMINOR) Logger.minor(this, "Filling request queue... (SSK="+isSSKScheduler+" insert="+isInsertScheduler); ChosenRequest req = null; synchronized(starterQueue) { From toad at freenetproject.org Tue Jul 1 13:15:06 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 13:15:06 +0000 (UTC) Subject: [freenet-cvs] r20906 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701131506.ED7F1479A2F@freenetproject.org> Author: toad Date: 2008-07-01 13:15:06 +0000 (Tue, 01 Jul 2008) New Revision: 20906 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java Log: Logging Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 12:58:33 UTC (rev 20905) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 13:15:06 UTC (rev 20906) @@ -856,6 +856,8 @@ public void run(ObjectContainer container, ClientContext context) { container.activate(req.request, 1); + if(logMINOR) + Logger.minor(this, "Requeueing "+req); if(req.request.isCancelled(container)) { container.delete(req); return; From toad at freenetproject.org Tue Jul 1 13:45:34 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 13:45:34 +0000 (UTC) Subject: [freenet-cvs] r20907 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701134534.41D2A479A53@freenetproject.org> Author: toad Date: 2008-07-01 13:45:33 +0000 (Tue, 01 Jul 2008) New Revision: 20907 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java Log: reregisterAll() - don't forget to reregister persistent requests Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 13:15:06 UTC (rev 20906) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 13:45:33 UTC (rev 20907) @@ -592,6 +592,7 @@ public void reregisterAll(final ClientRequester request, ObjectContainer container) { schedTransient.reregisterAll(request, random, this, container); + schedCore.reregisterAll(request, random, this, container); starter.wakeUp(); } From j16sdiz at freenetproject.org Tue Jul 1 14:03:39 2008 From: j16sdiz at freenetproject.org (j16sdiz at freenetproject.org) Date: Tue, 1 Jul 2008 14:03:39 +0000 (UTC) Subject: [freenet-cvs] r20908 - branches/saltedhashstore/freenet/src/freenet/store Message-ID: <20080701140339.0E5A0479A48@freenetproject.org> Author: j16sdiz Date: 2008-07-01 14:03:38 +0000 (Tue, 01 Jul 2008) New Revision: 20908 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: some old data lost on resize-resize-restart Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 13:45:33 UTC (rev 20907) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 14:03:38 UTC (rev 20908) @@ -131,7 +131,7 @@ if (maxKeys != storeSize) { if (prevStoreSize != 0) { - storeSize = prevStoreSize; + storeSize = Math.max(prevStoreSize, storeSize); prevStoreSize = 0; } setMaxKeys(maxKeys, true); From toad at freenetproject.org Tue Jul 1 14:15:28 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 14:15:28 +0000 (UTC) Subject: [freenet-cvs] r20909 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701141528.B731F479A58@freenetproject.org> Author: toad Date: 2008-07-01 14:15:28 +0000 (Tue, 01 Jul 2008) New Revision: 20909 Modified: branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java Log: regmeOnly only if specified Modified: branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java 2008-07-01 14:03:38 UTC (rev 20908) +++ branches/db4o/freenet/src/freenet/client/async/SplitFileFetcher.java 2008-07-01 14:15:28 UTC (rev 20909) @@ -333,7 +333,7 @@ regmeOnly = true; if(Logger.shouldLog(Logger.MINOR, this)) Logger.minor(this, "Scheduling "+this); for(int i=0;i Author: toad Date: 2008-07-01 14:35:13 +0000 (Tue, 01 Jul 2008) New Revision: 20910 Modified: branches/db4o/freenet/src/freenet/client/async/ClientGetter.java branches/db4o/freenet/src/freenet/client/async/SimpleManifestPutter.java Log: MUST update database after cancelling! Modified: branches/db4o/freenet/src/freenet/client/async/ClientGetter.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientGetter.java 2008-07-01 14:15:28 UTC (rev 20909) +++ branches/db4o/freenet/src/freenet/client/async/ClientGetter.java 2008-07-01 14:35:13 UTC (rev 20910) @@ -214,6 +214,8 @@ super.cancel(); s = currentState; } + if(persistent()) + container.set(this); if(s != null) { if(logMINOR) Logger.minor(this, "Cancelling "+currentState); if(persistent()) Modified: branches/db4o/freenet/src/freenet/client/async/SimpleManifestPutter.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/SimpleManifestPutter.java 2008-07-01 14:15:28 UTC (rev 20909) +++ branches/db4o/freenet/src/freenet/client/async/SimpleManifestPutter.java 2008-07-01 14:35:13 UTC (rev 20910) @@ -626,6 +626,8 @@ running = (PutHandler[]) runningPutHandlers.toArray(new PutHandler[runningPutHandlers.size()]); finished = true; } + if(persistent()) + container.set(this); for(int i=0;i Author: j16sdiz Date: 2008-07-01 14:36:00 +0000 (Tue, 01 Jul 2008) New Revision: 20911 Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java Log: resizeStore/rebuildBloom in one go Modified: branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java =================================================================== --- branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 14:35:13 UTC (rev 20910) +++ branches/saltedhashstore/freenet/src/freenet/store/SaltedHashFreenetStore.java 2008-07-01 14:36:00 UTC (rev 20911) @@ -961,20 +961,37 @@ initOldEntriesFile(); + configLock.writeLock().lock(); + try { + generation++; + bloomFilter.fork(); + keyCount.set(0); + } finally { + configLock.writeLock().unlock(); + } + final List oldEntryList = new LinkedList(); // start from end of store, make store shrinking quicker long startOffset = (_prevStoreSize / RESIZE_MEMORY_ENTRIES) * RESIZE_MEMORY_ENTRIES; int i = 0; for (long curOffset = startOffset; curOffset >= 0; curOffset -= RESIZE_MEMORY_ENTRIES) { - if (shutdown || _prevStoreSize != prevStoreSize) + if (shutdown || _prevStoreSize != prevStoreSize) { + bloomFilter.discard(); return; + } batchProcessEntries(curOffset, RESIZE_MEMORY_ENTRIES, new BatchProcessor() { public Entry process(Entry entry) { - if (entry.getStoreSize() == storeSize) // new size - return NOT_MODIFIED; + entry.setGeneration(generation); + keyCount.incrementAndGet(); + if (entry.getStoreSize() == storeSize) {// new size + bloomFilter.updateFilter(entry.getDigestedRoutingKey()); + + return entry; + } + oldEntryList.add(entry); return null; } @@ -1011,7 +1028,9 @@ try { if (_prevStoreSize != prevStoreSize) return; + bloomFilter.merge(); prevStoreSize = 0; + flags &= ~FLAG_REBUILD_BLOOM; } finally { configLock.writeLock().unlock(); } @@ -1198,7 +1217,7 @@ try { if (!isFree(offset) && Arrays.equals(getDigestedKeyFromOffset(offset), entry.getDigestedRoutingKey())) { - writeEntry(entry, offset); // overwrite, don't update key count + // do nothing return true; } } catch (IOException e) { @@ -1211,6 +1230,7 @@ try { if (isFree(offset)) { writeEntry(entry, offset); + bloomFilter.updateFilter(entry.getDigestedRoutingKey()); keyCount.incrementAndGet(); return true; } @@ -1307,7 +1327,6 @@ prevStoreSize = storeSize; storeSize = newStoreSize; - flags |= FLAG_REBUILD_BLOOM; writeConfigFile(); } finally { configLock.writeLock().unlock(); From toad at freenetproject.org Tue Jul 1 14:53:51 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 14:53:51 +0000 (UTC) Subject: [freenet-cvs] r20912 - branches/db4o/freenet/src/freenet/support/io Message-ID: <20080701145351.21209479A06@freenetproject.org> Author: toad Date: 2008-07-01 14:53:50 +0000 (Tue, 01 Jul 2008) New Revision: 20912 Modified: branches/db4o/freenet/src/freenet/support/io/BaseFileBucket.java Log: Logging Modified: branches/db4o/freenet/src/freenet/support/io/BaseFileBucket.java =================================================================== --- branches/db4o/freenet/src/freenet/support/io/BaseFileBucket.java 2008-07-01 14:36:00 UTC (rev 20911) +++ branches/db4o/freenet/src/freenet/support/io/BaseFileBucket.java 2008-07-01 14:53:50 UTC (rev 20912) @@ -384,6 +384,8 @@ public void free(boolean forceFree) { Object[] toClose; + if(Logger.shouldLog(Logger.MINOR, this)) + Logger.minor(this, "Freeing "+this); synchronized(this) { if(freed) return; freed = true; From toad at freenetproject.org Tue Jul 1 15:09:18 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 15:09:18 +0000 (UTC) Subject: [freenet-cvs] r20913 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701150918.0A7FF4799DD@freenetproject.org> Author: toad Date: 2008-07-01 15:09:17 +0000 (Tue, 01 Jul 2008) New Revision: 20913 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java Log: Ignore cancelled altReq's. Logging. Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 14:53:50 UTC (rev 20912) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 15:09:17 UTC (rev 20913) @@ -407,6 +407,10 @@ } if(altReq != null) container.activate(altReq, 1); + if(altReq != null && altReq.isCancelled(container)) { + if(logMINOR) + Logger.minor(this, "Ignoring cancelled recently succeeded item "+altReq); + } if(altReq != null && altReq.getPriorityClass(container) <= choosenPriorityClass && fixRetryCount(altReq.getRetryCount()) <= chosenTracker.getNumber() && !altReq.isEmpty(container)) { // Use the recent one instead @@ -550,6 +554,8 @@ } RegisterMe reg = new RegisterMe(req, req.getPriorityClass(container), this); container.set(reg); + if(logMINOR) + Logger.minor(this, "Queued RegisterMe for "+req+" : "+reg); return reg; } From toad at freenetproject.org Tue Jul 1 16:01:57 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 16:01:57 +0000 (UTC) Subject: [freenet-cvs] r20914 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701160157.A89E2479A17@freenetproject.org> Author: toad Date: 2008-07-01 16:01:57 +0000 (Tue, 01 Jul 2008) New Revision: 20914 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java Log: doh Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 15:09:17 UTC (rev 20913) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 16:01:57 UTC (rev 20914) @@ -410,6 +410,7 @@ if(altReq != null && altReq.isCancelled(container)) { if(logMINOR) Logger.minor(this, "Ignoring cancelled recently succeeded item "+altReq); + altReq = null; } if(altReq != null && altReq.getPriorityClass(container) <= choosenPriorityClass && fixRetryCount(altReq.getRetryCount()) <= chosenTracker.getNumber() && !altReq.isEmpty(container)) { From toad at freenetproject.org Tue Jul 1 16:03:12 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 16:03:12 +0000 (UTC) Subject: [freenet-cvs] r20915 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701160312.57ADC479A1C@freenetproject.org> Author: toad Date: 2008-07-01 16:03:12 +0000 (Tue, 01 Jul 2008) New Revision: 20915 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java Log: Check cancelled || empty here Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 16:01:57 UTC (rev 20914) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 16:03:12 UTC (rev 20915) @@ -407,7 +407,7 @@ } if(altReq != null) container.activate(altReq, 1); - if(altReq != null && altReq.isCancelled(container)) { + if(altReq != null && (altReq.isCancelled(container) || altReq.isEmpty(container))) { if(logMINOR) Logger.minor(this, "Ignoring cancelled recently succeeded item "+altReq); altReq = null; From toad at freenetproject.org Tue Jul 1 16:31:28 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 16:31:28 +0000 (UTC) Subject: [freenet-cvs] r20916 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701163128.7D2EB479A26@freenetproject.org> Author: toad Date: 2008-07-01 16:31:28 +0000 (Tue, 01 Jul 2008) New Revision: 20916 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerBase.java branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java Log: Recognise when a selected request is not in pendingKeys for the key and complain. Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerBase.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerBase.java 2008-07-01 16:03:12 UTC (rev 20915) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerBase.java 2008-07-01 16:31:28 UTC (rev 20916) @@ -224,6 +224,23 @@ } } + protected boolean inPendingKeys(SendableRequest req, Key key) { + Object o; + synchronized(pendingKeys) { + o = pendingKeys.get(key); + } + if(o == null) { + return false; + } else if(o instanceof SendableGet) { + return o == req; + } else { + SendableGet[] gets = (SendableGet[]) o; + for(int i=0;i Author: toad Date: 2008-07-01 16:36:38 +0000 (Tue, 01 Jul 2008) New Revision: 20917 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java Log: doh Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 16:31:28 UTC (rev 20916) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 16:36:38 UTC (rev 20917) @@ -267,7 +267,7 @@ container.set(ret); if(logMINOR) Logger.minor(this, "Storing "+ret); - if((req instanceof SendableGet) && inPendingKeys(req, key)) { + if((req instanceof SendableGet) && !inPendingKeys(req, key)) { Logger.error(this, "Selected key not in pendingKeys: key "+key+" for "+req); } } else { From toad at freenetproject.org Tue Jul 1 17:14:16 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 17:14:16 +0000 (UTC) Subject: [freenet-cvs] r20918 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701171416.667A4479A2F@freenetproject.org> Author: toad Date: 2008-07-01 17:14:16 +0000 (Tue, 01 Jul 2008) New Revision: 20918 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java Log: tripPending() and succeeded() must run at the same high priority, because if succeeded() completes first we end up rerunning the request (through the recently-succeeded mechanism). Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 16:36:38 UTC (rev 20917) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestScheduler.java 2008-07-01 17:14:16 UTC (rev 20918) @@ -507,6 +507,7 @@ ChosenRequest req = null; synchronized(starterQueue) { int size = starterQueue.size(); + if(logMINOR) Logger.minor(this, "Queue size: "+size); if(size >= MAX_STARTER_QUEUE_SIZE) { if(size >= WARNING_STARTER_QUEUE_SIZE) Logger.error(this, "Queue already full: "+starterQueue.size()); @@ -600,6 +601,14 @@ return choosenPriorityScheduler; } + /* + * tripPendingKey() callbacks must run quickly, since we've found a block. + * succeeded() must run quickly, since we delete the PersistentChosenRequest. + * tripPendingKey() must run before succeeded() so we don't choose the same + * request again, then remove it from pendingKeys before it completes! + */ + static final short TRIP_PENDING_PRIORITY = NativeThread.HIGH_PRIORITY-1; + public synchronized void succeeded(final BaseSendableGet succeeded, final ChosenRequest req) { if(req.isPersistent()) { jobRunner.queue(new DBJob() { @@ -609,7 +618,7 @@ container.delete((PersistentChosenRequest)req); } - }, NativeThread.HIGH_PRIORITY-1, false); + }, TRIP_PENDING_PRIORITY, false); // Boost the priority so the PersistentChosenRequest gets deleted reasonably quickly. } else schedTransient.succeeded(succeeded, null); @@ -674,7 +683,7 @@ if(logMINOR) Logger.minor(this, "Finished running callbacks"); } - }, NativeThread.NORM_PRIORITY, false); + }, TRIP_PENDING_PRIORITY, false); } From toad at freenetproject.org Tue Jul 1 17:14:58 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 17:14:58 +0000 (UTC) Subject: [freenet-cvs] r20919 - branches/db4o/freenet/src/freenet/client/async Message-ID: <20080701171458.9EC59479A1C@freenetproject.org> Author: toad Date: 2008-07-01 17:14:58 +0000 (Tue, 01 Jul 2008) New Revision: 20919 Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java Log: Reduce latency penalty from big RegisterMe's (i.e. splitfile segments) Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 17:14:16 UTC (rev 20918) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequestSchedulerCore.java 2008-07-01 17:14:58 UTC (rev 20919) @@ -511,6 +511,7 @@ class RegisterMeRunner implements DBJob { public void run(ObjectContainer container, ClientContext context) { + long deadline = System.currentTimeMillis() + 10*1000; for(int i=0;i < 10; i++) { try { if(!registerMeSet.hasNext()) break; @@ -544,6 +545,7 @@ // Cancel the request, and commit so it isn't tried again. reg.getter.internalError(null, t, sched, container, context); } + if(System.currentTimeMillis() > deadline) break; } if(registerMeSet.hasNext()) context.jobRunner.queue(registerMeRunner, NativeThread.NORM_PRIORITY, true); From toad at freenetproject.org Tue Jul 1 17:31:43 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 17:31:43 +0000 (UTC) Subject: [freenet-cvs] r20920 - branches/db4o/freenet/src/freenet/node/fcp Message-ID: <20080701173143.95CEA479A46@freenetproject.org> Author: toad Date: 2008-07-01 17:31:43 +0000 (Tue, 01 Jul 2008) New Revision: 20920 Modified: branches/db4o/freenet/src/freenet/node/fcp/RemovePersistentRequest.java Log: Boost priority of RemovePersistentRequest. It is interactive. Modified: branches/db4o/freenet/src/freenet/node/fcp/RemovePersistentRequest.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/RemovePersistentRequest.java 2008-07-01 17:14:58 UTC (rev 20919) +++ branches/db4o/freenet/src/freenet/node/fcp/RemovePersistentRequest.java 2008-07-01 17:31:43 UTC (rev 20920) @@ -60,7 +60,7 @@ } } - }, NativeThread.NORM_PRIORITY, false); + }, NativeThread.HIGH_PRIORITY, false); } } From toad at freenetproject.org Tue Jul 1 17:34:54 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 17:34:54 +0000 (UTC) Subject: [freenet-cvs] r20921 - branches/db4o/freenet/src/freenet/node/fcp Message-ID: <20080701173454.70A07479A44@freenetproject.org> Author: toad Date: 2008-07-01 17:34:54 +0000 (Tue, 01 Jul 2008) New Revision: 20921 Modified: branches/db4o/freenet/src/freenet/node/fcp/RemovePersistentRequest.java Log: Fix RemovePersistentRequest Modified: branches/db4o/freenet/src/freenet/node/fcp/RemovePersistentRequest.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/RemovePersistentRequest.java 2008-07-01 17:31:43 UTC (rev 20920) +++ branches/db4o/freenet/src/freenet/node/fcp/RemovePersistentRequest.java 2008-07-01 17:34:54 UTC (rev 20921) @@ -50,7 +50,7 @@ public void run(ObjectContainer container, ClientContext context) { try { - ClientRequest req = handler.removePersistentRebootRequest(global, identifier); + ClientRequest req = handler.removePersistentForeverRequest(global, identifier, container); if(req == null) { Logger.error(this, "Huh ? the request is null!"); } From toad at freenetproject.org Tue Jul 1 18:01:48 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Tue, 1 Jul 2008 18:01:48 +0000 (UTC) Subject: [freenet-cvs] r20922 - in branches/db4o/freenet/src/freenet: client client/async node/fcp node/updater Message-ID: <20080701180148.1B9E8479A40@freenetproject.org> Author: toad Date: 2008-07-01 18:01:47 +0000 (Tue, 01 Jul 2008) New Revision: 20922 Modified: branches/db4o/freenet/src/freenet/client/HighLevelSimpleClientImpl.java branches/db4o/freenet/src/freenet/client/async/ClientRequester.java branches/db4o/freenet/src/freenet/client/async/SimpleHealingQueue.java branches/db4o/freenet/src/freenet/client/async/SimpleManifestPutter.java branches/db4o/freenet/src/freenet/client/async/USKFetcherWrapper.java branches/db4o/freenet/src/freenet/client/async/USKRetriever.java branches/db4o/freenet/src/freenet/node/fcp/ClientGet.java branches/db4o/freenet/src/freenet/node/fcp/ClientPutBase.java branches/db4o/freenet/src/freenet/node/fcp/ClientPutDir.java branches/db4o/freenet/src/freenet/node/fcp/ClientRequest.java branches/db4o/freenet/src/freenet/node/fcp/FCPClient.java branches/db4o/freenet/src/freenet/node/fcp/FCPConnectionHandler.java branches/db4o/freenet/src/freenet/node/fcp/FCPServer.java branches/db4o/freenet/src/freenet/node/updater/NodeUpdater.java branches/db4o/freenet/src/freenet/node/updater/RevocationChecker.java Log: Maybe make cancelling/removing requests work Modified: branches/db4o/freenet/src/freenet/client/HighLevelSimpleClientImpl.java =================================================================== --- branches/db4o/freenet/src/freenet/client/HighLevelSimpleClientImpl.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/client/HighLevelSimpleClientImpl.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -244,7 +244,7 @@ core.getTicker().queueTimedJob(new Runnable() { public void run() { - get.cancel(); + get.cancel(null, core.clientContext); } }, timeout); Modified: branches/db4o/freenet/src/freenet/client/async/ClientRequester.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/ClientRequester.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/client/async/ClientRequester.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -33,9 +33,11 @@ hashCode = super.hashCode(); // the old object id will do fine, as long as we ensure it doesn't change! } - public synchronized void cancel() { + synchronized void cancel() { cancelled = true; } + + public abstract void cancel(ObjectContainer container, ClientContext context); public boolean isCancelled() { return cancelled; @@ -106,6 +108,7 @@ Logger.minor(this, "Completed block ("+dontNotify+ "): total="+totalBlocks+" success="+successfulBlocks+" failed="+failedBlocks+" fatally="+fatallyFailedBlocks+" finalised="+blockSetFinalized+" required="+minSuccessBlocks+" on "+this); synchronized(this) { successfulBlocks++; + if(cancelled) return; if(dontNotify) return; } if(persistent()) container.set(this); Modified: branches/db4o/freenet/src/freenet/client/async/SimpleHealingQueue.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/SimpleHealingQueue.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/client/async/SimpleHealingQueue.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -130,4 +130,8 @@ // Ignore } + public void cancel(ObjectContainer container, ClientContext context) { + super.cancel(); + } + } Modified: branches/db4o/freenet/src/freenet/client/async/SimpleManifestPutter.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/SimpleManifestPutter.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/client/async/SimpleManifestPutter.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -83,6 +83,10 @@ container.set(this); } + public void cancel(ObjectContainer container, ClientContext context) { + super.cancel(); + } + public FreenetURI getURI() { return null; } @@ -634,7 +638,7 @@ } } - public void cancel(ObjectContainer container) { + public void cancel(ObjectContainer container, ClientContext context) { super.cancel(); if(persistent()) container.set(this); Modified: branches/db4o/freenet/src/freenet/client/async/USKFetcherWrapper.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/USKFetcherWrapper.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/client/async/USKFetcherWrapper.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -66,4 +66,8 @@ public void onFinalizedMetadata(ObjectContainer container) { // Ignore } + + public void cancel(ObjectContainer container, ClientContext context) { + super.cancel(); + } } Modified: branches/db4o/freenet/src/freenet/client/async/USKRetriever.java =================================================================== --- branches/db4o/freenet/src/freenet/client/async/USKRetriever.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/client/async/USKRetriever.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -106,4 +106,8 @@ return cb.getPollingPriorityProgress(); } + public void cancel(ObjectContainer container, ClientContext context) { + super.cancel(); + } + } Modified: branches/db4o/freenet/src/freenet/node/fcp/ClientGet.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/ClientGet.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/node/fcp/ClientGet.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -332,9 +332,9 @@ container.set(this); // Update } - public void onLostConnection(ObjectContainer container) { + public void onLostConnection(ObjectContainer container, ClientContext context) { if(persistenceType == PERSIST_CONNECTION) - cancel(null); + cancel(container, context); // Otherwise ignore } Modified: branches/db4o/freenet/src/freenet/node/fcp/ClientPutBase.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/ClientPutBase.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/node/fcp/ClientPutBase.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -131,9 +131,9 @@ } } - public void onLostConnection(ObjectContainer container) { + public void onLostConnection(ObjectContainer container, ClientContext context) { if(persistenceType == PERSIST_CONNECTION) - cancel(container); + cancel(container, context); // otherwise ignore } Modified: branches/db4o/freenet/src/freenet/node/fcp/ClientPutDir.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/ClientPutDir.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/node/fcp/ClientPutDir.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -238,9 +238,9 @@ } } - public void onLostConnection(ObjectContainer container) { + public void onLostConnection(ObjectContainer container, ClientContext context) { if(persistenceType == PERSIST_CONNECTION) - cancel(container); + cancel(container, context); // otherwise ignore } Modified: branches/db4o/freenet/src/freenet/node/fcp/ClientRequest.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/ClientRequest.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/node/fcp/ClientRequest.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -122,7 +122,7 @@ } /** Lost connection */ - public abstract void onLostConnection(ObjectContainer container); + public abstract void onLostConnection(ObjectContainer container, ClientContext context); /** Send any pending messages for a persistent request e.g. after reconnecting */ public abstract void sendPendingMessages(FCPConnectionOutputHandler handler, boolean includePersistentRequest, boolean includeData, boolean onlyData, ObjectContainer container); @@ -207,10 +207,10 @@ } } - public void cancel(ObjectContainer container) { + public void cancel(ObjectContainer container, ClientContext context) { ClientRequester cr = getClientRequest(); // It might have been finished on startup. - if(cr != null) cr.cancel(); + if(cr != null) cr.cancel(container, context); freeData(container); if(persistenceType == PERSIST_FOREVER) container.set(this); @@ -237,8 +237,8 @@ protected abstract ClientRequester getClientRequest(); /** Completed request dropped off the end without being acknowledged */ - public void dropped(ObjectContainer container) { - cancel(container); + public void dropped(ObjectContainer container, ClientContext context) { + cancel(container, context); freeData(container); } Modified: branches/db4o/freenet/src/freenet/node/fcp/FCPClient.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/FCPClient.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/node/fcp/FCPClient.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -178,7 +178,7 @@ } } - public boolean removeByIdentifier(String identifier, boolean kill, FCPServer server, ObjectContainer container) { + public boolean removeByIdentifier(String identifier, boolean kill, FCPServer server, ObjectContainer container, ClientContext context) { assert((persistenceType == ClientRequest.PERSIST_FOREVER) == (container != null)); ClientRequest req; boolean logMINOR = Logger.shouldLog(Logger.MINOR, this); @@ -196,7 +196,7 @@ req.requestWasRemoved(container); if(kill) { if(logMINOR) Logger.minor(this, "Killing request "+req); - req.cancel(container); + req.cancel(container, context); } if(completionCallback != null) completionCallback.onRemove(req, container); @@ -368,7 +368,7 @@ container.delete(this); } - public void removeAll(ObjectContainer container) { + public void removeAll(ObjectContainer container, ClientContext context) { HashSet toKill = new HashSet(); synchronized(this) { Iterator i = runningPersistentRequests.iterator(); @@ -396,7 +396,7 @@ Iterator i = toStart.iterator(); while(i.hasNext()) { ClientRequest req = (ClientRequest) i.next(); - req.cancel(container); + req.cancel(container, context); req.requestWasRemoved(container); } } Modified: branches/db4o/freenet/src/freenet/node/fcp/FCPConnectionHandler.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/FCPConnectionHandler.java 2008-07-01 17:34:54 UTC (rev 20921) +++ branches/db4o/freenet/src/freenet/node/fcp/FCPConnectionHandler.java 2008-07-01 18:01:47 UTC (rev 20922) @@ -111,7 +111,7 @@ dupe = killedDupe; } for(int i=0;i Author: toad Date: 2008-07-02 12:45:46 +0000 (Wed, 02 Jul 2008) New Revision: 20923 Modified: branches/db4o/freenet/src/freenet/client/InsertBlock.java Log: ,1 isn't enough for a URI Modified: branches/db4o/freenet/src/freenet/client/InsertBlock.java =================================================================== --- branches/db4o/freenet/src/freenet/client/InsertBlock.java 2008-07-01 18:01:47 UTC (rev 20922) +++ branches/db4o/freenet/src/freenet/client/InsertBlock.java 2008-07-02 12:45:46 UTC (rev 20923) @@ -48,7 +48,7 @@ public void objectOnActivate(ObjectContainer container) { // Cascading activation of dependancies container.activate(data, 1); // will cascade - container.activate(desiredURI, 1); + container.activate(desiredURI, 5); } } From toad at freenetproject.org Wed Jul 2 12:52:21 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Wed, 2 Jul 2008 12:52:21 +0000 (UTC) Subject: [freenet-cvs] r20924 - branches/db4o/freenet/src/freenet/support/io Message-ID: <20080702125221.5277B479B3E@freenetproject.org> Author: toad Date: 2008-07-02 12:52:21 +0000 (Wed, 02 Jul 2008) New Revision: 20924 Modified: branches/db4o/freenet/src/freenet/support/io/ReadOnlyFileSliceBucket.java Log: ReadOnlyFileSliceBucket activation Modified: branches/db4o/freenet/src/freenet/support/io/ReadOnlyFileSliceBucket.java =================================================================== --- branches/db4o/freenet/src/freenet/support/io/ReadOnlyFileSliceBucket.java 2008-07-02 12:45:46 UTC (rev 20923) +++ branches/db4o/freenet/src/freenet/support/io/ReadOnlyFileSliceBucket.java 2008-07-02 12:52:21 UTC (rev 20924) @@ -155,4 +155,10 @@ container.delete(file); container.delete(this); } + + public void objectOnActivate(ObjectContainer container) { + // Cascading activation of dependancies + container.activate(file, 5); + } + } From toad at freenetproject.org Wed Jul 2 13:16:57 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Wed, 2 Jul 2008 13:16:57 +0000 (UTC) Subject: [freenet-cvs] r20925 - in branches/db4o/freenet/src/freenet: clients/http node/fcp Message-ID: <20080702131657.EEF8B479A31@freenetproject.org> Author: toad Date: 2008-07-02 13:16:57 +0000 (Wed, 02 Jul 2008) New Revision: 20925 Modified: branches/db4o/freenet/src/freenet/clients/http/QueueToadlet.java branches/db4o/freenet/src/freenet/node/fcp/ClientRequest.java branches/db4o/freenet/src/freenet/node/fcp/FCPServer.java Log: Don't forget to register() persistent puts and putdirs from the queue page. Modified: branches/db4o/freenet/src/freenet/clients/http/QueueToadlet.java =================================================================== --- branches/db4o/freenet/src/freenet/clients/http/QueueToadlet.java 2008-07-02 12:52:21 UTC (rev 20924) +++ branches/db4o/freenet/src/freenet/clients/http/QueueToadlet.java 2008-07-02 13:16:57 UTC (rev 20925) @@ -265,7 +265,13 @@ // FIXME should this be a proper localised message? It shouldn't happen... but we'd like to get reports if it does. } if(clientPut != null) - fcp.startBlocking(clientPut); + try { + fcp.startBlocking(clientPut); + } catch (IdentifierCollisionException e) { + Logger.error(this, "Cannot put same file twice in same millisecond"); + writePermanentRedirect(ctx, "Done", "/queue/"); + return; + } writePermanentRedirect(ctx, "Done", "/queue/"); return; } else if (request.isPartSet("insert-local-file")) { @@ -310,7 +316,14 @@ return; // FIXME should this be a proper localised message? It shouldn't happen... but we'd like to get reports if it does. } - fcp.startBlocking(clientPut); + if(clientPut != null) + try { + fcp.startBlocking(clientPut); + } catch (IdentifierCollisionException e) { + Logger.error(this, "Cannot put same file twice in same millisecond"); + writePermanentRedirect(ctx, "Done", "/queue/"); + return; + } writePermanentRedirect(ctx, "Done", "/queue/"); return; } else if (request.isPartSet("insert-local-dir")) { @@ -343,7 +356,14 @@ this.writeError(L10n.getString("QueueToadlet.errorNoFileOrCannotRead"), L10n.getString("QueueToadlet.errorAccessDeniedFile", new String[]{ "file" }, new String[]{ file.toString() }), ctx); return; } - fcp.startBlocking(clientPutDir); + if(clientPutDir != null) + try { + fcp.startBlocking(clientPutDir); + } catch (IdentifierCollisionException e) { + Logger.error(this, "Cannot put same file twice in same millisecond"); + writePermanentRedirect(ctx, "Done", "/queue/"); + return; + } writePermanentRedirect(ctx, "Done", "/queue/"); return; } Modified: branches/db4o/freenet/src/freenet/node/fcp/ClientRequest.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/ClientRequest.java 2008-07-02 12:52:21 UTC (rev 20924) +++ branches/db4o/freenet/src/freenet/node/fcp/ClientRequest.java 2008-07-02 13:16:57 UTC (rev 20925) @@ -206,6 +206,8 @@ return null; } } + + abstract void register(ObjectContainer container, boolean lazyResume, boolean noTags) throws IdentifierCollisionException; public void cancel(ObjectContainer container, ClientContext context) { ClientRequester cr = getClientRequest(); Modified: branches/db4o/freenet/src/freenet/node/fcp/FCPServer.java =================================================================== --- branches/db4o/freenet/src/freenet/node/fcp/FCPServer.java 2008-07-02 12:52:21 UTC (rev 20924) +++ branches/db4o/freenet/src/freenet/node/fcp/FCPServer.java 2008-07-02 13:16:57 UTC (rev 20925) @@ -931,12 +931,13 @@ Logger.error(this, "Replacing request completion callback "+cb, new Exception("error")); } - public void startBlocking(final ClientRequest req) { + public void startBlocking(final ClientRequest req) throws IdentifierCollisionException { if(req.persistenceType == ClientRequest.PERSIST_REBOOT) { req.start(null, core.clientContext); } else { class OutputWrapper { boolean done; + IdentifierCollisionException collided; } final OutputWrapper ow = new OutputWrapper(); core.clientContext.jobRunner.queue(new DBJob() { @@ -944,7 +945,10 @@ public void run(ObjectContainer container, ClientContext context) { container.activate(req, 1); try { + req.register(container, false, false); req.start(container, context); + } catch (IdentifierCollisionException e) { + ow.collided = e; } finally { synchronized(ow) { ow.done = true; @@ -964,6 +968,8 @@ // Ignore } } else { + if(ow.collided != null) + throw ow.collided; return; } } From toad at freenetproject.org Wed Jul 2 13:39:57 2008 From: toad at freenetproject.org (toad at freenetproject.org) Date: Wed, 2 Jul 2008 13:39:57 +0000 (UTC) Subject: [freenet-cvs] r20926 - branches/db4o/freenet/src/freenet/client Message-ID: <20080702133957.17ADD479A58@freenetproject.org> Author: toad Date: 2008-07-02 13:39:56 +0000 (Wed, 02 Jul 2008) New Revision: 20926 Modified: branches/db4o/freenet/src/freenet/client/FECQueue.java Log: Activation Modified: branches/db4o/freenet/src/freenet/client/FECQueue.java =================================================================== --- branches/db4o/freenet/src/freenet/client/FECQueue.java 2008-07-02 13:16:57 UTC (rev 20925) +++ branches/db4o/freenet/src/freenet/client/FECQueue.java 2008-07-02 13:39:56 UTC (rev 20926) @@ -185,6 +185,7 @@ public void run(ObjectContainer container, ClientContext context) { if(Logger.shouldLog(Logger.MINOR, this)) Logger.minor(this, "Running callback for "+job); + container.activate(job, 2); if(job.isADecodingJob) job.callback.onDecodedSegment(container, clientContext, job, job.dataBlocks, job.checkBlocks, job.dataBlockStatus, job.checkBlockStatus); else From nextgens at freenetproject.org Wed Jul 2 14:08:34 2008 From: nextgens at freenetproject.org (nextgens at freenetproject.org) Date: Wed, 2 Jul 2008 14:08:34 +0000 (UTC) Subject: [freenet-cvs] r20927 - trunk/freenet/src/freenet/l10n Message-ID: <20080702140834.744C7479B4A@freenetproject.org> Author: nextgens Date: 2008-07-02 14:08:34 +0000 (Wed, 02 Jul 2008) New Revision: 20927 Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties Log: l10n: apply patch from Luke771 (ita: r20268) Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties =================================================================== --- trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties 2008-07-02 13:39:56 UTC (rev 20926) +++ trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties 2008-07-02 14:08:34 UTC (rev 20927) @@ -263,7 +263,7 @@ FProxyToadlet.alerts=Messaggi dettagliati sullo stato del nodo FProxyToadlet.alertsTitle=Avvertenze FProxyToadlet.backToFProxy=${link}Clicca qui${/link} per andare alla homepage di FProxy -FProxyToadlet.backToReferrer=${link}Clicka qui${/link} per tornare alla pagina referente. +FProxyToadlet.backToReferrer=${link}Clicca qui${/link} per tornare alla pagina referente. FProxyToadlet.cantBindPort=Non ? possibile abbinare FProxy a quella porta! FProxyToadlet.config=configura il nodo FProxyToadlet.configTitle=Configurazione @@ -297,10 +297,10 @@ FProxyToadlet.notEnoughMetaStrings=Non ci sono abbastanza meta-strings FProxyToadlet.notFoundTitle=Non Trovato FProxyToadlet.openAsText=${link}Clicca qui${/link} per aprire il file come testo 'liscio' (plain text) . Questo non dovrebbe essere pericoloso ma la visualizzazione pu? risultare disturbata. -FProxyToadlet.openForce=${link}Clicka qui${/link} per aprire il file come ${mime} (leggi l'avvertenza sopra!). -FProxyToadlet.openForceDisk=${link}Clicka qui${/link} per forzare il browser a scaricare il file su disco. +FProxyToadlet.openForce=${link}Clicca qui${/link} per aprire il file come ${mime} (leggi l'avvertenza sopra!). +FProxyToadlet.openForceDisk=${link}Clicca qui${/link} per forzare il browser a scaricare il file su disco. FProxyToadlet.openPossRSSAsForceDisk=${link}Clicka qui${/link} per scaricare il file su disco (${bold}possibile pericolo${/bold} se si sta usando Firefox 2.0.0.0; il problema dovrebbe essere stato risolto in 2.0.0.1 -FProxyToadlet.openPossRSSAsPlainText=${link}Clicka qui${/link} per aprire il file come testo 'liscio' (plain text) (ci? ${bold}pu? rappresentare un pericolo${/bold} se si sta usando IE7 o FF2). +FProxyToadlet.openPossRSSAsPlainText=${link}Clicca qui${/link} per aprire il file come testo 'liscio' (plain text) (ci? ${bold}pu? rappresentare un pericolo${/bold} se si sta usando IE7 o FF2). FProxyToadlet.openPossRSSForceDisk=${link}Clicca qui${/link} per cercare di costringere il browser a scaricare il file su disco (${bold}Pu? essere pericoloso se si sta usando Firefox 2.0.0${/bold} ; il problema dovrebbe essere risolto in 2.0.1). FProxyToadlet.openRSSAsRSS=${link}Clicca qui${/link} per aprire il file come RSS (ci? ${bold}rappresenta un pericolo${/bold} se l'autore del sito ? mal intenzionato, perch? il filtraggio di RSS in Freenet non ? ancora stato implementato). FProxyToadlet.openRSSForce=${link}Clicca qui${/link} per aprire il file come ${mime} (ci? ${bold}pu? costituire un pericolo${/bold} se si usa IE7 o FF2). @@ -575,7 +575,7 @@ JPEGFilter.tooShortTitle=Troppo corto KnownUnsafeContentTypeException.dangerousInlines=Contenuto di questo tipo pu? includere immagini o video che vengono caricati direttamente da internet, esponendo l'indirizzo IP dell'utente e quindi compromettendone gravemente l'anonimato. KnownUnsafeContentTypeException.dangerousInlinesLabel=Collegamenti esterni pericolosi: -KnownUnsafeContentTypeException.dangerousLinks=Contenuto di questo tipo pu? includere dei link a pagine pubblicate al di fuori di Freenet. Clickando su tali link (e possono essere camuffati) si espone il proprio indirizzo IP, compromettendo l'anonimato. +KnownUnsafeContentTypeException.dangerousLinks=Contenuto di questo tipo pu? includere dei link a pagine pubblicate al di fuori di Freenet. Cliccando su tali link (e possono essere camuffati) si espone il proprio indirizzo IP, compromettendo l'anonimato. KnownUnsafeContentTypeException.dangerousLinksLabel=Link pericolosi: KnownUnsafeContentTypeException.dangerousMetadata=Contenuto di questo tipo pu? includere metadati, i quali possono essere visualizzati da alcuni browser o con l'aiuto di altri programmi: i metadati possono contenere link o immagini caricate direttamente da internet, che possono compromettere l'anonimato. KnownUnsafeContentTypeException.dangerousMetadataLabel=Metadati pericolosi: @@ -688,6 +688,9 @@ Node.inBWLimit=Limite ampiezza di banda in entrata (bytes per secondo) Node.inBWLimitLong=Limite dell'ampiezza di banda in entrata (bytes/sec); il nodo cerca di non eccedere tale limite; -1 siglifica quattro volte il limite impostato per l'ampiezza di banda in uscita (outputBandwidthLimit) Node.invalidStoreSize=Il datastore (magazzino dati) deve essere di dimensioni uguali o superiori a 32MB +Node.java14ShortText=Tra breve non sara pi? possibile operare un nodo Freenet su Java 1.4. Si prega di aggiornare a 1.5 o 1.6 +Node.java14Text=Freenet sta girando su una versione 1.4 di Java. Si prega di aggiornare almeno a Java 1.5. Gli aggiornamenti automatici di Freenet sono stati temporaneamente disabilitati in quanto versioni future potrebbero non non girare del tutto. +Node.java14Title=Freenet non ? pi? compatibile con Java 1.4 Node.l10nLanguage=Lingua dell' interfaccia grafica Node.l10nLanguageLong=Cambia la lingua in cui messaggi sono visualizzati. Alcune frasi e messaggi saranno visibili in versione tradotta solo dopo il prossimo riavvio del nodo. Node.maxHTL=HTL massimo @@ -808,6 +811,10 @@ OpennetUserAlert.warningTitle=Avvertenza: Modo Promiscuo Attivato: Il nodo si connetter? a degli sconosciuti PNGFilter.invalidHeader=Il file che si sta cercando di richiamare non e' un PNG: esso non include un header PNG valido. Potrebbe trattarsi di un file in altro formato e il browser potrebbe fare qualcosa di pericoloso a causa della confusione creata dalla mancata corrispondenza; il file ? stato pertanto bloccato. PNGFilter.invalidHeaderTitle=Header PNG non valido +PageMaker.modeAdvanced=Interfaccia avanzata +PageMaker.modeAdvancedTooltip=Un'interfaccia pi? complessa, utile soltanto agli utenti esperti +PageMaker.modeSimple=Interfaccia semplice +PageMaker.modeSimpleTooltip=Un'interfaccia semplice che pu? essere usata da ogni utente PeerManagerUserAlert.clockProblem=${count} peer non riescono a connettersi perch? il loro orologio di sistema differisce di oltre 24 ore da quello locale. Si prega di verificare che l'orologio di sistema del computer sia regolato sull'ora esatta. Regolazioni inaccurate dell'orologio causano molti problemi nel funzionamento dei meccanismi tra nodo e client. PeerManagerUserAlert.clockProblemTitle=Problema di sincronizzazione. PeerManagerUserAlert.connError=${count} peer non riescono a connettersi per motivi sconosciuti, forse per causa di 'bug' nel nodo o di una referenza di un nodo che e' corrotta. @@ -932,9 +939,12 @@ QueueToadlet.completedUDirectory=Directory Upload Completati:(${size}) QueueToadlet.delete=Elimina QueueToadlet.download=Scarica +QueueToadlet.downloadFilesInstructions=E' possibile incollare una serie di chiavi da scaricare nello spazio sottostante (una per rigo) QueueToadlet.downloadSucceeded=Il file ${origlink}${filename}${/origlink} ? stato scaricato. ${link}Click qui${/link} per aprile il file (${size}). QueueToadlet.downloadSucceededTitle=Download effettuato: ${filename} QueueToadlet.emergency=urgenza estrema +QueueToadlet.enqueuedFailure=Non ? stato possibile agguingere alla coda le ${number} chiavi qui elencate: +QueueToadlet.enqueuedSuccessfully=Le ${number} chiavi elencate di seguito sono state aggiunte alla coda con successo: QueueToadlet.errorAccessDenied=Errore: Accesso Negato QueueToadlet.errorAccessDeniedFile=L'attuale configurazione del nodo non consente l'upload del file "${file}". QueueToadlet.errorDToDisk=Non ? stato possibile scaricare su disco @@ -1149,7 +1159,7 @@ TimeSkewDetectedUserAlert.text=Una discrepanza temporale ? stata rilevata dal nodo. Questo ? un inconveniente grave; il nodo non potr? funzionare correttamente finch? non vi si sar? ovviato. Tra le cause pi? comuni, la modalit? powersafe mal configurata, cattiva sincronizzazione tra i client in rete, problemi di hardware. TimeSkewDetectedUserAlert.title=Discrepanza temporale rilevata! Toadlet.cancel=Cancella -Toadlet.clickHere=Clicka qui +Toadlet.clickHere=Clicca qui Toadlet.homepage=Homepage Toadlet.internalErrorPleaseReport=Errore interno: riportarlo se possibile Toadlet.internalErrorTitle=Errore Interno @@ -1162,6 +1172,7 @@ Toadlet.returnToHomepage=Torna alla homepage del nodo Toadlet.returnToNodeHomepage=Torna alla homepage del nodo Toadlet.returnToPrevPage=Torna alla pagina precedente +Toadlet.returnToQueuepage=Torna alla pagina della coda Toadlet.tempRedirectWithReason=Redirect temporaneo: ${reason} Toadlet.unauthorized=L'accesso a questa pagina ? interdetto. Toadlet.unauthorizedTitle=Non Autorizzato @@ -1180,7 +1191,7 @@ TranslationToadlet.confirmRemoveOverride=Conferma: elimina chiave di traduzione : (${key} - ${value}) ? TranslationToadlet.contributingToLabelWithLang=Stai contribuendo alla traduzione: ${lang}: TranslationToadlet.currentTranslationLabel=Traduzione corrente -TranslationToadlet.downloadTranslationsFile=Scarica file della traduzione eseguita +TranslationToadlet.downloadTranslationsFile=Scarica il file della traduzione TranslationToadlet.hideAlreadyTranslated=Nascondi stringhe tradotte TranslationToadlet.noCustomTranslations=Nessuna traduzione personalizzata disponibile. TranslationToadlet.originalVersionLabel=Originale (versione inglese) From nextgens at freenetproject.org Wed Jul 2 15:45:29 2008 From: nextgens at freenetproject.org (nextgens at freenetproject.org) Date: Wed, 2 Jul 2008 15:45:29 +0000 (UTC) Subject: [freenet-cvs] r20928 - trunk/freenet/src/freenet/clients/http/staticfiles Message-ID: <20080702154529.B1E74479B6C@freenetproject.org> Author: nextgens Date: 2008-07-02 15:45:29 +0000 (Wed, 02 Jul 2008) New Revision: 20928 Modified: trunk/freenet/src/freenet/clients/http/staticfiles/defaultbookmarks.dat Log: update edition numbers of the bookmarks Modified: trunk/freenet/src/freenet/clients/http/staticfiles/defaultbookmarks.dat =================================================================== --- trunk/freenet/src/freenet/clients/http/staticfiles/defau