[freenet-cvs] r14828 - in branches/freenet-jfk: . .settings src/freenet/client src/freenet/client/async src/freenet/clients/http src/freenet/clients/http/bookmark src/freenet/config src/freenet/crypt src/freenet/crypt/ciphers src/freenet/io src/freenet/io/comm src/freenet/io/xfer src/freenet/keys src/freenet/l10n src/freenet/pluginmanager src/freenet/store src/freenet/support src/freenet/support/compress src/freenet/support/io src/freenet/support/math src/freenet/tools src/org/spaceroots/mantissa/random test/freenet test/freenet/support test/freenet/utils
kryptos at freenetproject.org
kryptos at freenetproject.org
Tue Aug 21 20:27:01 UTC 2007
Author: kryptos
Date: 2007-08-21 20:26:59 +0000 (Tue, 21 Aug 2007)
New Revision: 14828
Added:
branches/freenet-jfk/src/freenet/client/ArchiveExtractCallback.java
branches/freenet-jfk/src/freenet/client/async/BlockSet.java
branches/freenet-jfk/src/freenet/client/async/SimpleBlockSet.java
branches/freenet-jfk/src/freenet/clients/http/ConnectionsToadlet.java
branches/freenet-jfk/src/freenet/clients/http/LinkEnabledCallback.java
branches/freenet-jfk/src/freenet/clients/http/OpennetConnectionsToadlet.java
branches/freenet-jfk/src/freenet/io/comm/MessageCore.java
branches/freenet-jfk/src/freenet/io/comm/PacketSocketHandler.java
branches/freenet-jfk/src/freenet/io/comm/SocketHandler.java
branches/freenet-jfk/src/freenet/io/comm/UdpSocketHandler.java
branches/freenet-jfk/src/freenet/l10n/freenet.l10n.de.properties
branches/freenet-jfk/src/freenet/pluginmanager/FredPluginHTTPAdvanced.java
branches/freenet-jfk/src/freenet/support/Executor.java
branches/freenet-jfk/src/freenet/support/HTMLEntities.java
branches/freenet-jfk/src/freenet/support/MutableBoolean.java
branches/freenet-jfk/src/freenet/support/PooledExecutor.java
branches/freenet-jfk/src/freenet/support/WeakHashSet.java
branches/freenet-jfk/src/freenet/support/io/BaseFileBucket.java
branches/freenet-jfk/src/freenet/support/io/MultiReaderBucket.java
branches/freenet-jfk/src/freenet/support/io/PersistentTempFileBucket.java
branches/freenet-jfk/src/freenet/tools/MergeSFS.java
branches/freenet-jfk/test/freenet/support/
branches/freenet-jfk/test/freenet/support/Base64Test.java
branches/freenet-jfk/test/freenet/support/BitArrayTest.java
branches/freenet-jfk/test/freenet/support/HTMLEncoderDecoderTest.java
branches/freenet-jfk/test/freenet/support/HTMLNodeTest.java
branches/freenet-jfk/test/freenet/support/HexUtilTest.java
branches/freenet-jfk/test/freenet/support/LRUHashtableTest.java
branches/freenet-jfk/test/freenet/support/LRUQueueTest.java
branches/freenet-jfk/test/freenet/support/MultiValueTableTest.java
branches/freenet-jfk/test/freenet/support/SimpleFieldSetTest.java
branches/freenet-jfk/test/freenet/support/SizeUtilTest.java
branches/freenet-jfk/test/freenet/support/TimeUtilTest.java
branches/freenet-jfk/test/freenet/support/URIPreEncoderTest.java
branches/freenet-jfk/test/freenet/support/URLEncoderDecoderTest.java
branches/freenet-jfk/test/freenet/utils/
branches/freenet-jfk/test/freenet/utils/UTFUtil.java
Removed:
branches/freenet-jfk/src/freenet/io/comm/UdpSocketManager.java
branches/freenet-jfk/src/freenet/support/io/FileBucketFactory.java
branches/freenet-jfk/src/freenet/support/io/RandomAccessFileBucket.java
branches/freenet-jfk/src/freenet/support/io/SpyInputStream.java
branches/freenet-jfk/src/freenet/support/io/SpyOutputStream.java
branches/freenet-jfk/src/freenet/support/io/TempBucketHook.java
branches/freenet-jfk/test/freenet/support/Base64Test.java
branches/freenet-jfk/test/freenet/support/BitArrayTest.java
branches/freenet-jfk/test/freenet/support/HTMLEncoderDecoderTest.java
branches/freenet-jfk/test/freenet/support/HTMLNodeTest.java
branches/freenet-jfk/test/freenet/support/HexUtilTest.java
branches/freenet-jfk/test/freenet/support/LRUHashtableTest.java
branches/freenet-jfk/test/freenet/support/LRUQueueTest.java
branches/freenet-jfk/test/freenet/support/MultiValueTableTest.java
branches/freenet-jfk/test/freenet/support/SimpleFieldSetTest.java
branches/freenet-jfk/test/freenet/support/SizeUtilTest.java
branches/freenet-jfk/test/freenet/support/TimeUtilTest.java
branches/freenet-jfk/test/freenet/support/URIPreEncoderTest.java
branches/freenet-jfk/test/freenet/support/URLEncoderDecoderTest.java
branches/freenet-jfk/test/freenet/utils/UTFUtil.java
Modified:
branches/freenet-jfk/.classpath
branches/freenet-jfk/.settings/org.eclipse.core.resources.prefs
branches/freenet-jfk/src/freenet/client/ArchiveManager.java
branches/freenet-jfk/src/freenet/client/ArchiveStoreContext.java
branches/freenet-jfk/src/freenet/client/ArchiveStoreItem.java
branches/freenet-jfk/src/freenet/client/ClientMetadata.java
branches/freenet-jfk/src/freenet/client/ErrorArchiveStoreItem.java
branches/freenet-jfk/src/freenet/client/FECCodec.java
branches/freenet-jfk/src/freenet/client/FetchContext.java
branches/freenet-jfk/src/freenet/client/FetchException.java
branches/freenet-jfk/src/freenet/client/HighLevelSimpleClientImpl.java
branches/freenet-jfk/src/freenet/client/InsertContext.java
branches/freenet-jfk/src/freenet/client/Metadata.java
branches/freenet-jfk/src/freenet/client/MetadataParseException.java
branches/freenet-jfk/src/freenet/client/RealArchiveStoreItem.java
branches/freenet-jfk/src/freenet/client/TempStoreElement.java
branches/freenet-jfk/src/freenet/client/async/BaseSingleFileFetcher.java
branches/freenet-jfk/src/freenet/client/async/BinaryBlob.java
branches/freenet-jfk/src/freenet/client/async/BinaryBlobFormatException.java
branches/freenet-jfk/src/freenet/client/async/BinaryBlobInserter.java
branches/freenet-jfk/src/freenet/client/async/ClientGetter.java
branches/freenet-jfk/src/freenet/client/async/ClientRequestScheduler.java
branches/freenet-jfk/src/freenet/client/async/SimpleManifestPutter.java
branches/freenet-jfk/src/freenet/client/async/SimpleSingleFileFetcher.java
branches/freenet-jfk/src/freenet/client/async/SingleBlockInserter.java
branches/freenet-jfk/src/freenet/client/async/SingleFileFetcher.java
branches/freenet-jfk/src/freenet/client/async/SingleFileInserter.java
branches/freenet-jfk/src/freenet/client/async/SplitFileFetcher.java
branches/freenet-jfk/src/freenet/client/async/SplitFileFetcherSegment.java
branches/freenet-jfk/src/freenet/client/async/SplitFileFetcherSubSegment.java
branches/freenet-jfk/src/freenet/client/async/USKChecker.java
branches/freenet-jfk/src/freenet/client/async/USKFetcher.java
branches/freenet-jfk/src/freenet/client/async/USKInserter.java
branches/freenet-jfk/src/freenet/client/async/USKManager.java
branches/freenet-jfk/src/freenet/client/async/USKProxyCompletionCallback.java
branches/freenet-jfk/src/freenet/clients/http/BookmarkEditorToadlet.java
branches/freenet-jfk/src/freenet/clients/http/BrowserTestToadlet.java
branches/freenet-jfk/src/freenet/clients/http/ConfigToadlet.java
branches/freenet-jfk/src/freenet/clients/http/DarknetConnectionsToadlet.java
branches/freenet-jfk/src/freenet/clients/http/FProxyToadlet.java
branches/freenet-jfk/src/freenet/clients/http/FirstTimeWizardToadlet.java
branches/freenet-jfk/src/freenet/clients/http/HTTPRequestImpl.java
branches/freenet-jfk/src/freenet/clients/http/LocalFileInsertToadlet.java
branches/freenet-jfk/src/freenet/clients/http/N2NTMToadlet.java
branches/freenet-jfk/src/freenet/clients/http/PageMaker.java
branches/freenet-jfk/src/freenet/clients/http/PluginToadlet.java
branches/freenet-jfk/src/freenet/clients/http/PproxyToadlet.java
branches/freenet-jfk/src/freenet/clients/http/QueueToadlet.java
branches/freenet-jfk/src/freenet/clients/http/SimpleToadletServer.java
branches/freenet-jfk/src/freenet/clients/http/StatisticsToadlet.java
branches/freenet-jfk/src/freenet/clients/http/SymlinkerToadlet.java
branches/freenet-jfk/src/freenet/clients/http/Toadlet.java
branches/freenet-jfk/src/freenet/clients/http/ToadletContainer.java
branches/freenet-jfk/src/freenet/clients/http/ToadletContextImpl.java
branches/freenet-jfk/src/freenet/clients/http/TranslationToadlet.java
branches/freenet-jfk/src/freenet/clients/http/TrivialToadlet.java
branches/freenet-jfk/src/freenet/clients/http/WelcomeToadlet.java
branches/freenet-jfk/src/freenet/clients/http/bookmark/BookmarkCategories.java
branches/freenet-jfk/src/freenet/clients/http/bookmark/BookmarkCategory.java
branches/freenet-jfk/src/freenet/clients/http/bookmark/BookmarkItem.java
branches/freenet-jfk/src/freenet/clients/http/bookmark/BookmarkItems.java
branches/freenet-jfk/src/freenet/clients/http/bookmark/BookmarkManager.java
branches/freenet-jfk/src/freenet/config/FreenetFilePersistentConfig.java
branches/freenet-jfk/src/freenet/config/PersistentConfig.java
branches/freenet-jfk/src/freenet/config/SubConfig.java
branches/freenet-jfk/src/freenet/crypt/DSAPrivateKey.java
branches/freenet-jfk/src/freenet/crypt/DiffieHellman.java
branches/freenet-jfk/src/freenet/crypt/KEProtocol.java
branches/freenet-jfk/src/freenet/crypt/PCFBMode.java
branches/freenet-jfk/src/freenet/crypt/SHA256.java
branches/freenet-jfk/src/freenet/crypt/Yarrow.java
branches/freenet-jfk/src/freenet/crypt/ciphers/Rijndael_Algorithm.java
branches/freenet-jfk/src/freenet/crypt/ciphers/Rijndael_Properties.java
branches/freenet-jfk/src/freenet/io/Inet6AddressMatcher.java
branches/freenet-jfk/src/freenet/io/NetworkInterface.java
branches/freenet-jfk/src/freenet/io/comm/DMT.java
branches/freenet-jfk/src/freenet/io/comm/FreenetInetAddress.java
branches/freenet-jfk/src/freenet/io/comm/IOStatisticCollector.java
branches/freenet-jfk/src/freenet/io/comm/MessageFilter.java
branches/freenet-jfk/src/freenet/io/comm/MessageType.java
branches/freenet-jfk/src/freenet/io/comm/Peer.java
branches/freenet-jfk/src/freenet/io/comm/PeerContext.java
branches/freenet-jfk/src/freenet/io/comm/RetrievalException.java
branches/freenet-jfk/src/freenet/io/xfer/BlockReceiver.java
branches/freenet-jfk/src/freenet/io/xfer/BlockTransmitter.java
branches/freenet-jfk/src/freenet/io/xfer/BulkReceiver.java
branches/freenet-jfk/src/freenet/io/xfer/BulkTransmitter.java
branches/freenet-jfk/src/freenet/io/xfer/PartiallyReceivedBulk.java
branches/freenet-jfk/src/freenet/keys/ClientCHK.java
branches/freenet-jfk/src/freenet/keys/ClientSSKBlock.java
branches/freenet-jfk/src/freenet/keys/FreenetURI.java
branches/freenet-jfk/src/freenet/keys/Key.java
branches/freenet-jfk/src/freenet/keys/NodeCHK.java
branches/freenet-jfk/src/freenet/keys/TooBigException.java
branches/freenet-jfk/src/freenet/keys/USK.java
branches/freenet-jfk/src/freenet/l10n/L10n.java
branches/freenet-jfk/src/freenet/l10n/freenet.l10n.en.properties
branches/freenet-jfk/src/freenet/l10n/freenet.l10n.fr.properties
branches/freenet-jfk/src/freenet/l10n/freenet.l10n.it.properties
branches/freenet-jfk/src/freenet/l10n/freenet.l10n.no.properties
branches/freenet-jfk/src/freenet/l10n/freenet.l10n.se.properties
branches/freenet-jfk/src/freenet/pluginmanager/AccessDeniedPluginHTTPException.java
branches/freenet-jfk/src/freenet/pluginmanager/DownloadPluginHTTPException.java
branches/freenet-jfk/src/freenet/pluginmanager/FredPlugin.java
branches/freenet-jfk/src/freenet/pluginmanager/NotFoundPluginHTTPException.java
branches/freenet-jfk/src/freenet/pluginmanager/PluginHTTPException.java
branches/freenet-jfk/src/freenet/pluginmanager/PluginHandler.java
branches/freenet-jfk/src/freenet/pluginmanager/PluginManager.java
branches/freenet-jfk/src/freenet/pluginmanager/RedirectPluginHTTPException.java
branches/freenet-jfk/src/freenet/store/BerkeleyDBFreenetStore.java
branches/freenet-jfk/src/freenet/support/BitArray.java
branches/freenet-jfk/src/freenet/support/Fields.java
branches/freenet-jfk/src/freenet/support/FileLoggerHook.java
branches/freenet-jfk/src/freenet/support/HTMLDecoder.java
branches/freenet-jfk/src/freenet/support/HTMLEncoder.java
branches/freenet-jfk/src/freenet/support/HTMLNode.java
branches/freenet-jfk/src/freenet/support/HexUtil.java
branches/freenet-jfk/src/freenet/support/LRUHashtable.java
branches/freenet-jfk/src/freenet/support/LRUQueue.java
branches/freenet-jfk/src/freenet/support/RandomGrabArray.java
branches/freenet-jfk/src/freenet/support/ShortBuffer.java
branches/freenet-jfk/src/freenet/support/SimpleFieldSet.java
branches/freenet-jfk/src/freenet/support/StringArray.java
branches/freenet-jfk/src/freenet/support/TimeUtil.java
branches/freenet-jfk/src/freenet/support/TokenBucket.java
branches/freenet-jfk/src/freenet/support/URLDecoder.java
branches/freenet-jfk/src/freenet/support/compress/GzipCompressor.java
branches/freenet-jfk/src/freenet/support/io/ArrayBucket.java
branches/freenet-jfk/src/freenet/support/io/FileBucket.java
branches/freenet-jfk/src/freenet/support/io/FileUtil.java
branches/freenet-jfk/src/freenet/support/io/FilenameGenerator.java
branches/freenet-jfk/src/freenet/support/io/NullPersistentFileTracker.java
branches/freenet-jfk/src/freenet/support/io/PaddedEphemerallyEncryptedBucket.java
branches/freenet-jfk/src/freenet/support/io/PaddedEphemerallyEncryptedBucketFactory.java
branches/freenet-jfk/src/freenet/support/io/PersistentFileTracker.java
branches/freenet-jfk/src/freenet/support/io/PersistentTempBucketFactory.java
branches/freenet-jfk/src/freenet/support/io/RandomAccessFileWrapper.java
branches/freenet-jfk/src/freenet/support/io/RandomAccessThing.java
branches/freenet-jfk/src/freenet/support/io/SerializableToFieldSetBucketUtil.java
branches/freenet-jfk/src/freenet/support/io/TempBucketFactory.java
branches/freenet-jfk/src/freenet/support/io/TempFileBucket.java
branches/freenet-jfk/src/freenet/support/math/TimeDecayingRunningAverage.java
branches/freenet-jfk/src/org/spaceroots/mantissa/random/MersenneTwister.java
Log:
Merged with trunk(r14796) and r13448:Link level encryption using JFK
Modified: branches/freenet-jfk/.classpath
===================================================================
--- branches/freenet-jfk/.classpath 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/.classpath 2007-08-21 20:26:59 UTC (rev 14828)
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
- <classpathentry excluding="freenet/node/*Test.java|org/spaceroots/mantissa/random/MersenneTwisterTest.java|plugins/JSTUN/**|test/**" kind="src" path="src"/>
- <classpathentry including="freenet/**/*java" kind="src" path="test"/>
+ <classpathentry excluding="freenet/node/*Test.java|plugins/JSTUN/**|test/**" kind="src" path="src"/>
+ <classpathentry including="freenet/|org/" kind="src" path="test"/>
<classpathentry exported="true" kind="lib" path="lib/freenet-ext.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="/usr/share/java/junit.jar"/>
Modified: branches/freenet-jfk/.settings/org.eclipse.core.resources.prefs
===================================================================
--- branches/freenet-jfk/.settings/org.eclipse.core.resources.prefs 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/.settings/org.eclipse.core.resources.prefs 2007-08-21 20:26:59 UTC (rev 14828)
@@ -1,5 +1,6 @@
-#Thu May 31 21:45:29 BST 2007
+#Fri Jul 27 18:21:13 BST 2007
eclipse.preferences.version=1
+encoding//src/freenet/l10n/freenet.l10n.de.properties=UTF-8
encoding//src/freenet/l10n/freenet.l10n.en.properties=UTF-8
encoding//src/freenet/l10n/freenet.l10n.fr.properties=UTF-8
encoding//src/freenet/l10n/freenet.l10n.it.properties=UTF-8
Copied: branches/freenet-jfk/src/freenet/client/ArchiveExtractCallback.java (from rev 14796, trunk/freenet/src/freenet/client/ArchiveExtractCallback.java)
===================================================================
--- branches/freenet-jfk/src/freenet/client/ArchiveExtractCallback.java (rev 0)
+++ branches/freenet-jfk/src/freenet/client/ArchiveExtractCallback.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -0,0 +1,15 @@
+package freenet.client;
+
+import freenet.support.api.Bucket;
+
+/** Called when we have extracted an archive, and a specified file either is
+ * or isn't in it. */
+public interface ArchiveExtractCallback {
+
+ /** Got the data */
+ public void gotBucket(Bucket data);
+
+ /** Not in the archive */
+ public void notInArchive();
+
+}
Modified: branches/freenet-jfk/src/freenet/client/ArchiveManager.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/ArchiveManager.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/ArchiveManager.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -17,11 +17,12 @@
import freenet.keys.FreenetURI;
import freenet.support.LRUHashtable;
import freenet.support.Logger;
+import freenet.support.MutableBoolean;
import freenet.support.api.Bucket;
import freenet.support.io.BucketTools;
-import freenet.support.io.FileBucket;
import freenet.support.io.FilenameGenerator;
import freenet.support.io.PaddedEphemerallyEncryptedBucket;
+import freenet.support.io.TempFileBucket;
/**
* Cache of recently decoded archives:
@@ -34,11 +35,33 @@
*/
public class ArchiveManager {
+ public static final String METADATA_NAME = ".metadata";
private static boolean logMINOR;
+ final RandomSource random;
+ final long maxArchiveSize;
+ final long maxArchivedFileSize;
+
+ // ArchiveHandler's
+ final int maxArchiveHandlers;
+ private final LRUHashtable archiveHandlers;
+
+ // Data cache
+ /** Maximum number of cached ArchiveStoreItems */
+ final int maxCachedElements;
+ /** Maximum cached data in bytes */
+ final long maxCachedData;
+ /** Currently cached data in bytes */
+ private long cachedData;
+ /** Map from ArchiveKey to ArchiveStoreElement */
+ private final LRUHashtable storedData;
+ /** Filename generator */
+ private final FilenameGenerator filenameGenerator;
+
/**
* Create an ArchiveManager.
- * @param maxHandlers The maximum number of cached ArchiveHandler's.
+ * @param maxHandlers The maximum number of cached ArchiveHandler's i.e. the
+ * maximum number of containers to track.
* @param maxCachedData The maximum size of the cache directory, in bytes.
* @param maxArchiveSize The maximum size of an archive.
* @param maxArchivedFileSize The maximum extracted size of a single file in any
@@ -52,7 +75,6 @@
public ArchiveManager(int maxHandlers, long maxCachedData, long maxArchiveSize, long maxArchivedFileSize, int maxCachedElements, RandomSource random, FilenameGenerator filenameGenerator) {
maxArchiveHandlers = maxHandlers;
archiveHandlers = new LRUHashtable();
- cachedElements = new LRUHashtable();
this.maxCachedElements = maxCachedElements;
this.maxCachedData = maxCachedData;
storedData = new LRUHashtable();
@@ -63,16 +85,6 @@
logMINOR = Logger.shouldLog(Logger.MINOR, this);
}
- final RandomSource random;
- final long maxArchiveSize;
- final long maxArchivedFileSize;
-
-
- // ArchiveHandler's
-
- final int maxArchiveHandlers;
- final LRUHashtable archiveHandlers;
-
/** Add an ArchiveHandler by key */
private synchronized void putCached(FreenetURI key, ArchiveHandler zip) {
if(logMINOR) Logger.minor(this, "Put cached AH for "+key+" : "+zip);
@@ -90,24 +102,6 @@
return handler;
}
- // Element cache
-
- /** Cache of ArchiveElement's by MyKey */
- final LRUHashtable cachedElements;
- /** Maximum number of cached ArchiveElement's */
- final int maxCachedElements;
-
- // Data cache
-
- /** Maximum cached data in bytes */
- final long maxCachedData;
- /** Currently cached data in bytes */
- private long cachedData;
- /** Map from ArchiveKey to ArchiveStoreElement */
- final LRUHashtable storedData;
- /** Filename generator */
- final FilenameGenerator filenameGenerator;
-
/**
* Create an archive handler. This does not need to know how to
* fetch the key, because the methods called later will ask.
@@ -138,24 +132,30 @@
if(logMINOR) Logger.minor(this, "Fetch cached: "+key+ ' ' +filename);
ArchiveKey k = new ArchiveKey(key, filename);
ArchiveStoreItem asi = null;
- synchronized (storedData) {
+ synchronized (this) {
asi = (ArchiveStoreItem) storedData.get(k);
if(asi == null) return null;
// Promote to top of LRU
storedData.push(k, asi);
}
if(logMINOR) Logger.minor(this, "Found data");
- return asi.getDataOrThrow();
+ return asi.getReaderBucket();
}
/**
- * Remove a file from the cache.
+ * Remove a file from the cache. Called after it has been removed from its
+ * ArchiveHandler.
* @param item The ArchiveStoreItem to remove.
*/
- void removeCachedItem(ArchiveStoreItem item) {
- synchronized (storedData) {
- storedData.removeKey(item.key);
- }
+ synchronized void removeCachedItem(ArchiveStoreItem item) {
+ long size = item.spaceUsed();
+ storedData.removeKey(item.key);
+ // Hard disk space limit = remove it here.
+ // Soft disk space limit would be to remove it outside the lock.
+ // Soft disk space limit = we go over the limit significantly when we
+ // are overloaded.
+ cachedData -= size;
+ item.close();
}
/**
@@ -165,15 +165,19 @@
* @param data The actual data fetched.
* @param archiveContext The context for the whole fetch process.
* @param ctx The ArchiveStoreContext for this key.
+ * @param element A particular element that the caller is especially interested in, or null.
+ * @param callback A callback to be called if we find that element, or if we don't.
* @throws ArchiveFailureException If we could not extract the data, or it was too big, etc.
* @throws ArchiveRestartException
* @throws ArchiveRestartException If the request needs to be restarted because the archive
* changed.
*/
- public void extractToCache(FreenetURI key, short archiveType, Bucket data, ArchiveContext archiveContext, ArchiveStoreContext ctx) throws ArchiveFailureException, ArchiveRestartException {
+ public void extractToCache(FreenetURI key, short archiveType, Bucket data, ArchiveContext archiveContext, ArchiveStoreContext ctx, String element, ArchiveExtractCallback callback) throws ArchiveFailureException, ArchiveRestartException {
logMINOR = Logger.shouldLog(Logger.MINOR, this);
+ MutableBoolean gotElement = element != null ? new MutableBoolean() : null;
+
if(logMINOR) Logger.minor(this, "Extracting "+key);
ctx.onExtract();
ctx.removeAllCachedItems(); // flush cache anyway
@@ -211,7 +215,7 @@
// MINOR: Assumes the first entry in the zip is a directory.
ZipEntry entry;
- byte[] buf = new byte[4096];
+ byte[] buf = new byte[32768];
HashSet names = new HashSet();
boolean gotMetadata = false;
@@ -249,16 +253,22 @@
out.close();
if(name.equals(".metadata"))
gotMetadata = true;
- addStoreElement(ctx, key, name, temp);
+ addStoreElement(ctx, key, name, temp, gotElement, element, callback);
names.add(name);
+ trimStoredData();
}
}
// If no metadata, generate some
if(!gotMetadata) {
- generateMetadata(ctx, key, names);
+ generateMetadata(ctx, key, names, gotElement, element, callback);
+ trimStoredData();
}
if(throwAtExit) throw new ArchiveRestartException("Archive changed on re-fetch");
+
+ if((!gotElement.value) && element != null)
+ callback.notInArchive();
+
} catch (IOException e) {
throw new ArchiveFailureException("Error reading archive: "+e.getMessage(), e);
} finally {
@@ -277,9 +287,12 @@
* @param ctx The context object.
* @param key The key from which the archive we are unpacking was fetched.
* @param names Set of names in the archive.
+ * @param element2
+ * @param gotElement
+ * @param callbackName If we generate a
* @throws ArchiveFailureException
*/
- private void generateMetadata(ArchiveStoreContext ctx, FreenetURI key, HashSet names) throws ArchiveFailureException {
+ private ArchiveStoreItem generateMetadata(ArchiveStoreContext ctx, FreenetURI key, HashSet names, MutableBoolean gotElement, String element2, ArchiveExtractCallback callback) throws ArchiveFailureException {
/* What we have to do is to:
* - Construct a filesystem tree of the names.
* - Turn each level of the tree into a Metadata object, including those below it, with
@@ -304,21 +317,21 @@
OutputStream os = element.bucket.getOutputStream();
os.write(buf);
os.close();
- addStoreElement(ctx, key, ".metadata", element);
- break;
+ return addStoreElement(ctx, key, ".metadata", element, gotElement, element2, callback);
} catch (MetadataUnresolvedException e) {
try {
- x = resolve(e, x, element, ctx, key);
+ x = resolve(e, x, element, ctx, key, gotElement, element2, callback);
} catch (IOException e1) {
throw new ArchiveFailureException("Failed to create metadata: "+e1, e1);
}
} catch (IOException e1) {
+ Logger.error(this, "Failed to create metadata: "+e1, e1);
throw new ArchiveFailureException("Failed to create metadata: "+e1, e1);
}
}
}
- private int resolve(MetadataUnresolvedException e, int x, TempStoreElement element, ArchiveStoreContext ctx, FreenetURI key) throws IOException {
+ private int resolve(MetadataUnresolvedException e, int x, TempStoreElement element, ArchiveStoreContext ctx, FreenetURI key, MutableBoolean gotElement, String element2, ArchiveExtractCallback callback) throws IOException, ArchiveFailureException {
Metadata[] m = e.mustResolve;
for(int i=0;i<m.length;i++) {
try {
@@ -326,9 +339,9 @@
OutputStream os = element.bucket.getOutputStream();
os.write(buf);
os.close();
- addStoreElement(ctx, key, ".metadata-"+(x++), element);
+ addStoreElement(ctx, key, ".metadata-"+(x++), element, gotElement, element2, callback);
} catch (MetadataUnresolvedException e1) {
- x = resolve(e, x, element, ctx, key);
+ x = resolve(e, x, element, ctx, key, gotElement, element2, callback);
}
}
return x;
@@ -350,7 +363,7 @@
} else
after = name.substring(x+1, name.length());
Object o = dir.get(before);
- HashMap map;
+ HashMap map = (HashMap) o;
if(o == null) {
map = new HashMap();
dir.put(before, map);
@@ -358,7 +371,6 @@
if(o instanceof String) {
throw new ArchiveFailureException("Invalid archive: contains "+name+" as both file and dir");
}
- map = (HashMap) o;
addToDirectory(map, after, prefix + before + '/');
}
}
@@ -375,21 +387,51 @@
private void addErrorElement(ArchiveStoreContext ctx, FreenetURI key, String name, String error) {
ErrorArchiveStoreItem element = new ErrorArchiveStoreItem(ctx, key, name, error);
if(logMINOR) Logger.minor(this, "Adding error element: "+element+" for "+key+ ' ' +name);
- synchronized (storedData) {
+ ArchiveStoreItem oldItem;
+ synchronized (this) {
+ oldItem = (ArchiveStoreItem) storedData.get(element.key);
storedData.push(element.key, element);
+ if(oldItem != null) {
+ oldItem.close();
+ cachedData -= oldItem.spaceUsed();
+ }
}
}
/**
* Add a store element.
+ * @param callbackName If set, the name of the file for which we must call the callback if this file happens to
+ * match.
+ * @param gotElement Flag indicating whether we've already found the file for the callback. If so we must not call
+ * it again.
+ * @param callback Callback to be called if we do find it. We must getReaderBucket() before adding the data to the
+ * LRU, otherwise it may be deleted before it reaches the client.
+ * @throws ArchiveFailureException If a failure occurred resulting in the data not being readable. Only happens if
+ * callback != null.
*/
- private void addStoreElement(ArchiveStoreContext ctx, FreenetURI key, String name, TempStoreElement temp) {
- RealArchiveStoreItem element = new RealArchiveStoreItem(this, ctx, key, name, temp);
+ private ArchiveStoreItem addStoreElement(ArchiveStoreContext ctx, FreenetURI key, String name, TempStoreElement temp, MutableBoolean gotElement, String callbackName, ArchiveExtractCallback callback) throws ArchiveFailureException {
+ RealArchiveStoreItem element = new RealArchiveStoreItem(ctx, key, name, temp);
if(logMINOR) Logger.minor(this, "Adding store element: "+element+" ( "+key+ ' ' +name+" size "+element.spaceUsed()+" )");
- synchronized (storedData) {
+ ArchiveStoreItem oldItem;
+ // Let it throw, if it does something is drastically wrong
+ Bucket matchBucket = null;
+ if((!gotElement.value) && name.equals(callbackName)) {
+ matchBucket = element.getReaderBucket();
+ }
+ synchronized (this) {
+ oldItem = (ArchiveStoreItem) storedData.get(element.key);
storedData.push(element.key, element);
+ cachedData += element.spaceUsed();
+ if(oldItem != null) {
+ cachedData -= oldItem.spaceUsed();
+ oldItem.close();
+ }
}
- trimStoredData();
+ if(matchBucket != null) {
+ callback.gotBucket(matchBucket);
+ gotElement.value = true;
+ }
+ return element;
}
/**
@@ -397,15 +439,25 @@
* Call synchronized on storedData.
*/
private void trimStoredData() {
+ synchronized(this) {
while(true) {
- synchronized(this) {
+ ArchiveStoreItem item;
if(cachedData <= maxCachedData && storedData.size() <= maxCachedElements) return;
- }
- ArchiveStoreItem e = (ArchiveStoreItem) storedData.popValue();
+ if(storedData.isEmpty()) {
+ // Race condition? cachedData out of sync?
+ Logger.error(this, "storedData is empty but still over limit: cachedData="+cachedData+" / "+maxCachedData);
+ return;
+ }
+ item = (ArchiveStoreItem) storedData.popValue();
+ long space = item.spaceUsed();
+ cachedData -= space;
+ // Hard limits = delete file within lock, soft limits = delete outside of lock
+ // Here we use a hard limit
if(logMINOR)
- Logger.minor(this, "Dropping "+e+" : cachedData="+cachedData+" of "+maxCachedData);
- e.close();
+ Logger.minor(this, "Dropping "+item+" : cachedData="+cachedData+" of "+maxCachedData+" stored items : "+storedData.size()+" of "+maxCachedElements);
+ item.close();
}
+ }
}
/**
@@ -415,12 +467,13 @@
* go over the maximum size. Will obviously keep its key when we move it to main.
*/
private TempStoreElement makeTempStoreBucket(long size) {
- File myFile = filenameGenerator.makeRandomFilename();
- FileBucket fb = new FileBucket(myFile, false, false, true, true, true);
+ long id = filenameGenerator.makeRandomFilename();
+ File myFile = filenameGenerator.getFilename(id);
+ TempFileBucket fb = new TempFileBucket(id, filenameGenerator);
byte[] cipherKey = new byte[32];
random.nextBytes(cipherKey);
- PaddedEphemerallyEncryptedBucket encryptedBucket = new PaddedEphemerallyEncryptedBucket(fb, 1024, random, true);
+ PaddedEphemerallyEncryptedBucket encryptedBucket = new PaddedEphemerallyEncryptedBucket(fb, 1024, random);
return new TempStoreElement(myFile, fb, encryptedBucket);
}
@@ -440,12 +493,4 @@
return Metadata.ARCHIVE_ZIP;
else throw new IllegalArgumentException();
}
-
- synchronized void decrementSpace(long spaceUsed) {
- cachedData -= spaceUsed;
- }
-
- synchronized void incrementSpace(long spaceUsed) {
- cachedData += spaceUsed;
- }
}
Modified: branches/freenet-jfk/src/freenet/client/ArchiveStoreContext.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/ArchiveStoreContext.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/ArchiveStoreContext.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -26,6 +26,15 @@
private FreenetURI key;
private short archiveType;
private boolean forceRefetchArchive;
+ /** Archive size */
+ private long lastSize = -1;
+ /** Archive hash */
+ private byte[] lastHash;
+ /** Index of still-cached ArchiveStoreItems with this key.
+ * Note that we never ever hold this and then take another lock! In particular
+ * we must not take the ArchiveManager lock while holding this lock. It must be
+ * the inner lock to avoid deadlocks. */
+ private final DoublyLinkedListImpl myItems;
public ArchiveStoreContext(ArchiveManager manager, FreenetURI key, short archiveType, boolean forceRefetchArchive) {
this.manager = manager;
@@ -46,6 +55,8 @@
/**
* Fetch a file in an archive.
+ * @return A Bucket containing the data. This will not be freed until the
+ * client is finished with it i.e. calls free() or it is finalized.
*/
public Bucket get(String internalName, ArchiveContext archiveContext, ClientMetadata dm, int recursionLevel,
boolean dontEnterImplicitArchives) throws ArchiveFailureException, ArchiveRestartException, MetadataParseException, FetchException {
@@ -67,9 +78,6 @@
return null;
}
- // Archive size
- long lastSize = -1;
-
/** Returns the size of the archive last time we fetched it, or -1 */
long getLastSize() {
return lastSize;
@@ -80,10 +88,7 @@
lastSize = size;
}
- // Archive hash
- byte[] lastHash;
-
/** Returns the hash of the archive last time we fetched it, or null */
public byte[] getLastHash() {
return lastHash;
@@ -93,11 +98,6 @@
public void setLastHash(byte[] realHash) {
lastHash = realHash;
}
-
- // Index of still-cached ArchiveStoreItems with this key
-
- /** Index of still-cached ArchiveStoreItems with this key */
- final DoublyLinkedListImpl myItems;
/**
* Remove all ArchiveStoreItems with this key from the cache.
@@ -110,7 +110,6 @@
}
if(item == null) break;
manager.removeCachedItem(item);
- item.context.removeItem(item);
}
}
@@ -121,14 +120,14 @@
}
}
- /** Notify that an archive store item with this key has been expelled from the cache. */
+ /** Notify that an archive store item with this key has been expelled from the
+ * cache. Remove it from our local cache and ask it to free the bucket if
+ * necessary. */
public void removeItem(ArchiveStoreItem item) {
- long spaceUsed = item.spaceUsed();
synchronized(myItems) {
if(myItems.remove(item) == null) return; // only removed once
}
item.innerClose();
- manager.decrementSpace(spaceUsed);
}
public short getArchiveType() {
@@ -139,10 +138,11 @@
return key;
}
- public void extractToCache(Bucket bucket, ArchiveContext actx) throws ArchiveFailureException, ArchiveRestartException {
- manager.extractToCache(key, archiveType, bucket, actx, this);
+ public void extractToCache(Bucket bucket, ArchiveContext actx, String element, ArchiveExtractCallback callback) throws ArchiveFailureException, ArchiveRestartException {
+ manager.extractToCache(key, archiveType, bucket, actx, this, element, callback);
}
+ /** Called just before extracting this container to the cache */
public void onExtract() {
forceRefetchArchive = false;
}
Modified: branches/freenet-jfk/src/freenet/client/ArchiveStoreItem.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/ArchiveStoreItem.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/ArchiveStoreItem.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -20,7 +20,10 @@
context.addItem(this);
}
- /** Delete any stored data on disk etc. Override in subtypes for specific cleanup. */
+ /** Delete any stored data on disk etc.
+ * Override in subtypes for specific cleanup.
+ * Will be called with locks held, so should only do low level operations
+ * such as deletes.. */
void innerClose() { } // override in subtypes for cleanup
/**
@@ -36,7 +39,14 @@
abstract Bucket getDataOrThrow() throws ArchiveFailureException;
/**
- * Return the amount of cache space used by the item.
+ * Return the amount of cache space used by the item. May be called inside
+ * locks so should not take any nontrivial locks or take long.
*/
abstract long spaceUsed();
+
+ /**
+ * Get the data as a Bucket, and guarantee that it won't be freed until the
+ * returned object is either finalized or freed.
+ */
+ abstract Bucket getReaderBucket() throws ArchiveFailureException;
}
Modified: branches/freenet-jfk/src/freenet/client/ClientMetadata.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/ClientMetadata.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/ClientMetadata.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -52,4 +52,18 @@
public String toString() {
return getMIMEType();
}
+
+ public void clear() {
+ mimeType = null;
+ }
+
+ public String getMIMETypeNoParams() {
+ String s = mimeType;
+ if(s == null) return null;
+ int i = s.indexOf(';');
+ if(i > -1) {
+ s = s.substring(i);
+ }
+ return s;
+ }
}
Modified: branches/freenet-jfk/src/freenet/client/ErrorArchiveStoreItem.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/ErrorArchiveStoreItem.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/ErrorArchiveStoreItem.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -35,5 +35,9 @@
public long spaceUsed() {
return 0;
}
+
+ Bucket getReaderBucket() throws ArchiveFailureException {
+ throw new ArchiveFailureException(error);
+ }
}
Modified: branches/freenet-jfk/src/freenet/client/FECCodec.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/FECCodec.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/FECCodec.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -343,7 +343,7 @@
synchronized (_awaitingJobs) {
if(fecRunnerThread == null) {
if(fecRunnerThread != null) Logger.error(FECCodec.class, "The callback died!! restarting a new one, please report that error.");
- fecRunnerThread = new Thread(fecRunner, "FEC Pool");
+ fecRunnerThread = new Thread(fecRunner, "FEC Pool "+(fecPoolCounter++));
fecRunnerThread.setDaemon(true);
fecRunnerThread.setPriority(Thread.MIN_PRIORITY);
@@ -361,6 +361,7 @@
private static final LinkedList _awaitingJobs = new LinkedList();
private static final FECRunner fecRunner = new FECRunner();
private static Thread fecRunnerThread;
+ private static int fecPoolCounter;
/**
* A private Thread started by {@link FECCodec}...
Modified: branches/freenet-jfk/src/freenet/client/FetchContext.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/FetchContext.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/FetchContext.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -3,11 +3,16 @@
* http://www.gnu.org/ for further details of the GPL. */
package freenet.client;
+import java.util.Set;
+
+import freenet.client.async.BlockSet;
import freenet.client.async.HealingQueue;
import freenet.client.async.USKManager;
import freenet.client.events.ClientEventProducer;
import freenet.client.events.SimpleEventProducer;
import freenet.crypt.RandomSource;
+import freenet.node.Ticker;
+import freenet.support.Executor;
import freenet.support.api.BucketFactory;
/** Context for a Fetcher. Contains all the settings a Fetcher needs to know about. */
@@ -45,6 +50,11 @@
public boolean returnZIPManifests;
public final HealingQueue healingQueue;
public final boolean ignoreTooManyPathComponents;
+ /** If set, contains a set of blocks to be consulted before checking the datastore. */
+ public BlockSet blocks;
+ public Set allowedMIMETypes;
+ public final Ticker ticker;
+ public final Executor executor;
public FetchContext(long curMaxLength,
long curMaxTempLength, int maxMetadataSize, int maxRecursionLevel, int maxArchiveRestarts, int maxArchiveLevels,
@@ -53,7 +63,10 @@
boolean allowSplitfiles, boolean followRedirects, boolean localRequestOnly,
int maxDataBlocksPerSegment, int maxCheckBlocksPerSegment,
RandomSource random, ArchiveManager archiveManager, BucketFactory bucketFactory,
- ClientEventProducer producer, boolean cacheLocalRequests, USKManager uskManager, HealingQueue hq, boolean ignoreTooManyPathComponents) {
+ ClientEventProducer producer, boolean cacheLocalRequests, USKManager uskManager,
+ HealingQueue hq, boolean ignoreTooManyPathComponents, Ticker ticker, Executor executor) {
+ this.ticker = ticker;
+ this.executor = executor;
this.maxOutputLength = curMaxLength;
this.uskManager = uskManager;
this.maxTempLength = curMaxTempLength;
@@ -85,8 +98,12 @@
this.eventProducer = ctx.eventProducer;
else
this.eventProducer = new SimpleEventProducer();
+ this.ticker = ctx.ticker;
+ this.executor = ctx.executor;
this.uskManager = ctx.uskManager;
this.ignoreTooManyPathComponents = ctx.ignoreTooManyPathComponents;
+ this.blocks = ctx.blocks;
+ this.allowedMIMETypes = ctx.allowedMIMETypes;
if(maskID == IDENTICAL_MASK) {
this.maxOutputLength = ctx.maxOutputLength;
this.maxMetadataSize = ctx.maxMetadataSize;
Modified: branches/freenet-jfk/src/freenet/client/FetchException.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/FetchException.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/FetchException.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -127,6 +127,18 @@
Logger.minor(this, "FetchException("+getMessage(mode)+"): "+t.getMessage(),t);
}
+ public FetchException(int mode, String reason, Throwable t) {
+ super(reason+" : "+getMessage(mode)+": "+t.getMessage());
+ extraMessage = t.getMessage();
+ this.mode = mode;
+ errorCodes = null;
+ initCause(t);
+ newURI = null;
+ expectedSize = -1;
+ if(Logger.shouldLog(Logger.MINOR, this))
+ Logger.minor(this, "FetchException("+getMessage(mode)+"): "+t.getMessage(),t);
+ }
+
public FetchException(int mode, FailureCodeTracker errorCodes) {
super(getMessage(mode));
extraMessage = null;
@@ -182,6 +194,18 @@
this.finalizedSizeAndMimeType = e.finalizedSizeAndMimeType;
}
+ public FetchException(FetchException e, FreenetURI uri) {
+ super(e.getMessage());
+ initCause(e);
+ this.mode = e.mode;
+ this.newURI = uri;
+ this.errorCodes = e.errorCodes;
+ this.expectedMimeType = e.expectedMimeType;
+ this.expectedSize = e.expectedSize;
+ this.extraMessage = e.extraMessage;
+ this.finalizedSizeAndMimeType = e.finalizedSizeAndMimeType;
+ }
+
public static String getShortMessage(int mode) {
String ret = L10n.getString("FetchException.shortError."+mode);
if(ret == null)
@@ -273,6 +297,10 @@
public static final int ARCHIVE_RESTART = 26;
/** There is a more recent version of the USK, ~= HTTP 301; FProxy will turn this into a 301 */
public static final int PERMANENT_REDIRECT = 27;
+ /** Requestor specified a list of allowed MIME types, and the key's type wasn't in the list */
+ public static final int WRONG_MIME_TYPE = 29;
+ /** A node killed the request because it had recently been tried and had DNFed */
+ public static final int RECENTLY_FAILED = 30;
/** Is an error fatal i.e. is there no point retrying? */
public boolean isFatal() {
@@ -307,6 +335,7 @@
case REJECTED_OVERLOAD:
case TRANSFER_FAILED:
case ALL_DATA_NOT_FOUND:
+ case RECENTLY_FAILED: // wait a bit, but fine
// Not usually fatal
case SPLITFILE_ERROR:
return false;
@@ -320,6 +349,7 @@
case CANCELLED:
case ARCHIVE_RESTART:
case PERMANENT_REDIRECT:
+ case WRONG_MIME_TYPE:
// Fatal
return true;
Modified: branches/freenet-jfk/src/freenet/client/HighLevelSimpleClientImpl.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/HighLevelSimpleClientImpl.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/HighLevelSimpleClientImpl.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -191,7 +191,7 @@
MAX_SPLITFILE_BLOCKS_PER_SEGMENT, MAX_SPLITFILE_CHECK_BLOCKS_PER_SEGMENT,
random, archiveManager, bucketFactory, globalEventProducer,
cacheLocalRequests, core.uskManager, healingQueue,
- forceDontIgnoreTooManyPathComponents ? false : core.ignoreTooManyPathComponents);
+ forceDontIgnoreTooManyPathComponents ? false : core.ignoreTooManyPathComponents, core.getTicker(), core.getExecutor());
}
public InsertContext getInsertContext(boolean forceNonPersistent) {
@@ -199,6 +199,6 @@
forceNonPersistent ? new NullPersistentFileTracker() : persistentFileTracker,
random, INSERT_RETRIES, CONSECUTIVE_RNFS_ASSUME_SUCCESS,
SPLITFILE_INSERT_THREADS, SPLITFILE_BLOCKS_PER_SEGMENT, SPLITFILE_CHECK_BLOCKS_PER_SEGMENT,
- globalEventProducer, cacheLocalRequests, core.uskManager, blockEncoder);
+ globalEventProducer, cacheLocalRequests, core.uskManager, blockEncoder, core.getExecutor());
}
}
Modified: branches/freenet-jfk/src/freenet/client/InsertContext.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/InsertContext.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/InsertContext.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -8,6 +8,7 @@
import freenet.client.events.ClientEventProducer;
import freenet.client.events.SimpleEventProducer;
import freenet.crypt.RandomSource;
+import freenet.support.Executor;
import freenet.support.api.BucketFactory;
import freenet.support.io.NullPersistentFileTracker;
import freenet.support.io.PersistentFileTracker;
@@ -32,10 +33,11 @@
public final boolean cacheLocalRequests;
public final USKManager uskManager;
public final BackgroundBlockEncoder backgroundBlockEncoder;
+ public final Executor executor;
public InsertContext(BucketFactory bf, BucketFactory persistentBF, PersistentFileTracker tracker, RandomSource random,
int maxRetries, int rnfsToSuccess, int maxThreads, int splitfileSegmentDataBlocks, int splitfileSegmentCheckBlocks,
- ClientEventProducer eventProducer, boolean cacheLocalRequests, USKManager uskManager, BackgroundBlockEncoder blockEncoder) {
+ ClientEventProducer eventProducer, boolean cacheLocalRequests, USKManager uskManager, BackgroundBlockEncoder blockEncoder, Executor executor) {
this.bf = bf;
this.persistentFileTracker = tracker;
this.persistentBucketFactory = persistentBF;
@@ -51,6 +53,7 @@
this.splitfileSegmentCheckBlocks = splitfileSegmentCheckBlocks;
this.cacheLocalRequests = cacheLocalRequests;
this.backgroundBlockEncoder = blockEncoder;
+ this.executor = executor;
}
public InsertContext(InsertContext ctx, SimpleEventProducer producer, boolean forceNonPersistent) {
@@ -69,6 +72,7 @@
this.splitfileSegmentCheckBlocks = ctx.splitfileSegmentCheckBlocks;
this.cacheLocalRequests = ctx.cacheLocalRequests;
this.backgroundBlockEncoder = ctx.backgroundBlockEncoder;
+ this.executor = ctx.executor;
}
public InsertContext(InsertContext ctx, SimpleEventProducer producer) {
@@ -87,6 +91,7 @@
this.splitfileSegmentCheckBlocks = ctx.splitfileSegmentCheckBlocks;
this.cacheLocalRequests = ctx.cacheLocalRequests;
this.backgroundBlockEncoder = ctx.backgroundBlockEncoder;
+ this.executor = ctx.executor;
}
}
Modified: branches/freenet-jfk/src/freenet/client/Metadata.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/Metadata.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/Metadata.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -157,8 +157,9 @@
}
/** Parse some metadata from a byte[].
- * @throws IOException If the data is incomplete, or something wierd happens. */
- private Metadata(byte[] data) throws IOException {
+ * @throws IOException If the data is incomplete, or something wierd happens.
+ * @throws MetadataParseException */
+ private Metadata(byte[] data) throws IOException, MetadataParseException {
this(new DataInputStream(new ByteArrayInputStream(data)), data.length);
}
@@ -848,6 +849,7 @@
unresolvedMetadata = new LinkedList();
for(int j=0;j<m.length;j++)
unresolvedMetadata.addFirst(m[j]);
+ kill = true;
}
}
if(kill) {
@@ -916,4 +918,10 @@
public boolean isResolved() {
return resolvedURI != null;
}
+
+ public void setArchiveManifest() {
+ archiveType = ArchiveManager.getArchiveType(clientMetadata.getMIMEType());
+ clientMetadata.clear();
+ documentType = ZIP_MANIFEST;
+ }
}
Modified: branches/freenet-jfk/src/freenet/client/MetadataParseException.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/MetadataParseException.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/MetadataParseException.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -6,7 +6,7 @@
import java.io.IOException;
/** Thrown when Metadata parse fails. */
-public class MetadataParseException extends IOException {
+public class MetadataParseException extends Exception {
private static final long serialVersionUID = 4910650977022715220L;
Modified: branches/freenet-jfk/src/freenet/client/RealArchiveStoreItem.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/RealArchiveStoreItem.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/RealArchiveStoreItem.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -7,16 +7,14 @@
import freenet.keys.FreenetURI;
import freenet.support.api.Bucket;
-import freenet.support.io.FileBucket;
import freenet.support.io.FileUtil;
-import freenet.support.io.PaddedEphemerallyEncryptedBucket;
+import freenet.support.io.MultiReaderBucket;
class RealArchiveStoreItem extends ArchiveStoreItem {
- private final ArchiveManager manager;
private final File myFilename;
- private final PaddedEphemerallyEncryptedBucket bucket;
- private final FileBucket underBucket;
+ private final MultiReaderBucket mb;
+ private final Bucket bucket;
private final long spaceUsed;
/**
@@ -26,15 +24,13 @@
* @param temp The TempStoreElement currently storing the data.
* @param manager The parent ArchiveManager within which this item is stored.
*/
- RealArchiveStoreItem(ArchiveManager manager, ArchiveStoreContext ctx, FreenetURI key2, String realName, TempStoreElement temp) {
+ RealArchiveStoreItem(ArchiveStoreContext ctx, FreenetURI key2, String realName, TempStoreElement temp) {
super(new ArchiveKey(key2, realName), ctx);
- this.manager = manager;
- this.bucket = temp.bucket;
- this.underBucket = temp.underBucket;
- underBucket.setReadOnly();
- this.myFilename = underBucket.getFile();
- spaceUsed = FileUtil.estimateUsage(myFilename, underBucket.size());
- this.manager.incrementSpace(spaceUsed);
+ mb = new MultiReaderBucket(temp.bucket);
+ this.bucket = mb.getReaderBucket();
+ temp.underBucket.setReadOnly();
+ this.myFilename = temp.underBucket.getFile();
+ spaceUsed = FileUtil.estimateUsage(myFilename, temp.underBucket.size());
}
/**
@@ -59,10 +55,14 @@
}
void innerClose() {
- underBucket.free();
+ bucket.free();
}
Bucket getDataOrThrow() throws ArchiveFailureException {
return dataAsBucket();
}
+
+ Bucket getReaderBucket() throws ArchiveFailureException {
+ return mb.getReaderBucket();
+ }
}
Modified: branches/freenet-jfk/src/freenet/client/TempStoreElement.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/TempStoreElement.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/TempStoreElement.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -5,15 +5,15 @@
import java.io.File;
-import freenet.support.io.FileBucket;
import freenet.support.io.PaddedEphemerallyEncryptedBucket;
+import freenet.support.io.TempFileBucket;
class TempStoreElement {
final File myFilename;
final PaddedEphemerallyEncryptedBucket bucket;
- final FileBucket underBucket;
+ final TempFileBucket underBucket;
- TempStoreElement(File myFile, FileBucket fb, PaddedEphemerallyEncryptedBucket encryptedBucket) {
+ TempStoreElement(File myFile, TempFileBucket fb, PaddedEphemerallyEncryptedBucket encryptedBucket) {
this.myFilename = myFile;
this.underBucket = fb;
this.bucket = encryptedBucket;
Modified: branches/freenet-jfk/src/freenet/client/async/BaseSingleFileFetcher.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/BaseSingleFileFetcher.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/BaseSingleFileFetcher.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -6,9 +6,11 @@
import freenet.client.FetchContext;
import freenet.keys.ClientKey;
import freenet.keys.ClientSSK;
+import freenet.keys.Key;
+import freenet.keys.KeyBlock;
+import freenet.keys.KeyVerifyException;
import freenet.node.SendableGet;
import freenet.support.Logger;
-import freenet.support.RandomGrabArray;
public abstract class BaseSingleFileFetcher extends SendableGet {
@@ -80,8 +82,7 @@
synchronized(this) {
cancelled = true;
}
- RandomGrabArray arr = getParentGrabArray();
- if(arr != null) arr.remove(this);
+ super.unregister();
}
public synchronized boolean isCancelled() {
@@ -101,4 +102,20 @@
return true;
}
+ public void onGotKey(Key key, KeyBlock block) {
+ synchronized(this) {
+ if(isCancelled()) return;
+ if(!key.equals(this.key.getNodeKey())) {
+ Logger.normal(this, "Got sent key "+key+" but want "+this.key+" for "+this);
+ return;
+ }
+ }
+ try {
+ onSuccess(Key.createKeyBlock(this.key, block), false, 0);
+ } catch (KeyVerifyException e) {
+ Logger.error(this, "onGotKey("+key+","+block+") got "+e+" for "+this, e);
+ // FIXME if we get rid of the direct route this must call onFailure()
+ }
+ }
+
}
Modified: branches/freenet-jfk/src/freenet/client/async/BinaryBlob.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/BinaryBlob.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/BinaryBlob.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -1,10 +1,15 @@
package freenet.client.async;
+import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.EOFException;
import java.io.IOException;
+import com.onionnetworks.util.FileUtil;
+
import freenet.keys.Key;
import freenet.keys.KeyBlock;
+import freenet.keys.KeyVerifyException;
public abstract class BinaryBlob {
@@ -46,4 +51,70 @@
writeBlobHeader(binaryBlobStream, BinaryBlob.BLOB_END, BinaryBlob.BLOB_END_VERSION, 0);
}
+ public static void readBinaryBlob(DataInputStream dis, BlockSet blocks, boolean tolerant) throws IOException, BinaryBlobFormatException {
+ long magic = dis.readLong();
+ if(magic != BinaryBlob.BINARY_BLOB_MAGIC)
+ throw new BinaryBlobFormatException("Bad magic");
+ short version = dis.readShort();
+ if(version != BinaryBlob.BINARY_BLOB_OVERALL_VERSION)
+ throw new BinaryBlobFormatException("Unknown overall version");
+
+ int i=0;
+ while(true) {
+ long blobLength;
+ try {
+ blobLength = dis.readInt() & 0xFFFFFFFFL;
+ } catch (EOFException e) {
+ // End of file
+ dis.close();
+ break;
+ }
+ short blobType = dis.readShort();
+ short blobVer = dis.readShort();
+
+ if(blobType == BinaryBlob.BLOB_END) {
+ dis.close();
+ break;
+ } else if(blobType == BinaryBlob.BLOB_BLOCK) {
+ if(blobVer != BinaryBlob.BLOB_BLOCK_VERSION)
+ // Even if tolerant, if we can't read a blob there probably isn't much we can do.
+ throw new BinaryBlobFormatException("Unknown block blob version");
+ if(blobLength < 9)
+ throw new BinaryBlobFormatException("Block blob too short");
+ short keyType = dis.readShort();
+ int keyLen = dis.readUnsignedByte();
+ int headersLen = dis.readUnsignedShort();
+ int dataLen = dis.readUnsignedShort();
+ int pubkeyLen = dis.readUnsignedShort();
+ int total = 9 + keyLen + headersLen + dataLen + pubkeyLen;
+ if(blobLength != total)
+ throw new BinaryBlobFormatException("Binary blob not same length as data: blobLength="+blobLength+" total="+total);
+ byte[] keyBytes = new byte[keyLen];
+ byte[] headersBytes = new byte[headersLen];
+ byte[] dataBytes = new byte[dataLen];
+ byte[] pubkeyBytes = new byte[pubkeyLen];
+ dis.readFully(keyBytes);
+ dis.readFully(headersBytes);
+ dis.readFully(dataBytes);
+ dis.readFully(pubkeyBytes);
+ KeyBlock block;
+ try {
+ block = Key.createBlock(keyType, keyBytes, headersBytes, dataBytes, pubkeyBytes);
+ } catch (KeyVerifyException e) {
+ throw new BinaryBlobFormatException("Invalid key: "+e.getMessage(), e);
+ }
+
+ blocks.add(block);
+
+ } else {
+ if(tolerant) {
+ FileUtil.skipFully(dis, blobLength);
+ } else {
+ throw new BinaryBlobFormatException("Unknown blob type: "+blobType);
+ }
+ }
+ i++;
+ }
+
+ }
}
Modified: branches/freenet-jfk/src/freenet/client/async/BinaryBlobFormatException.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/BinaryBlobFormatException.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/BinaryBlobFormatException.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -4,6 +4,11 @@
public class BinaryBlobFormatException extends Exception {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
public BinaryBlobFormatException(String message) {
super(message);
}
Modified: branches/freenet-jfk/src/freenet/client/async/BinaryBlobInserter.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/BinaryBlobInserter.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/BinaryBlobInserter.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -1,19 +1,16 @@
package freenet.client.async;
import java.io.DataInputStream;
-import java.io.EOFException;
import java.io.IOException;
+import java.util.Iterator;
import java.util.Vector;
-import com.onionnetworks.util.FileUtil;
-
import freenet.client.FailureCodeTracker;
import freenet.client.InsertContext;
import freenet.client.InsertException;
import freenet.keys.CHKBlock;
import freenet.keys.Key;
import freenet.keys.KeyBlock;
-import freenet.keys.KeyVerifyException;
import freenet.keys.SSKBlock;
import freenet.node.LowLevelPutException;
import freenet.node.SimpleSendableInsert;
@@ -44,74 +41,24 @@
this.parent = parent;
this.clientContext = clientContext;
this.errors = new FailureCodeTracker(true);
- Vector myInserters = new Vector();
DataInputStream dis = new DataInputStream(blob.getInputStream());
- long magic = dis.readLong();
- if(magic != BinaryBlob.BINARY_BLOB_MAGIC)
- throw new BinaryBlobFormatException("Bad magic");
- short version = dis.readShort();
- if(version != BinaryBlob.BINARY_BLOB_OVERALL_VERSION)
- throw new BinaryBlobFormatException("Unknown overall version");
- int i=0;
- while(true) {
- long blobLength;
- try {
- blobLength = dis.readInt() & 0xFFFFFFFFL;
- } catch (EOFException e) {
- // End of file
- dis.close();
- break;
- }
- short blobType = dis.readShort();
- short blobVer = dis.readShort();
-
- if(blobType == BinaryBlob.BLOB_END) {
- dis.close();
- break;
- } else if(blobType == BinaryBlob.BLOB_BLOCK) {
- if(blobVer != BinaryBlob.BLOB_BLOCK_VERSION)
- // Even if tolerant, if we can't read a blob there probably isn't much we can do.
- throw new BinaryBlobFormatException("Unknown block blob version");
- if(blobLength < 9)
- throw new BinaryBlobFormatException("Block blob too short");
- short keyType = dis.readShort();
- int keyLen = dis.readUnsignedByte();
- int headersLen = dis.readUnsignedShort();
- int dataLen = dis.readUnsignedShort();
- int pubkeyLen = dis.readUnsignedShort();
- int total = 9 + keyLen + headersLen + dataLen + pubkeyLen;
- if(blobLength != total)
- throw new BinaryBlobFormatException("Binary blob not same length as data: blobLength="+blobLength+" total="+total);
- byte[] keyBytes = new byte[keyLen];
- byte[] headersBytes = new byte[headersLen];
- byte[] dataBytes = new byte[dataLen];
- byte[] pubkeyBytes = new byte[pubkeyLen];
- dis.readFully(keyBytes);
- dis.readFully(headersBytes);
- dis.readFully(dataBytes);
- dis.readFully(pubkeyBytes);
- KeyBlock block;
- try {
- block = Key.createBlock(keyType, keyBytes, headersBytes, dataBytes, pubkeyBytes);
- } catch (KeyVerifyException e) {
- throw new BinaryBlobFormatException("Invalid key: "+e.getMessage(), e);
- }
-
- MySendableInsert inserter =
- new MySendableInsert(i, block, prioClass, getScheduler(block), clientContext);
-
- myInserters.add(inserter);
-
- } else {
- if(tolerant) {
- FileUtil.skipFully(dis, blobLength);
- } else {
- throw new BinaryBlobFormatException("Unknown blob type: "+blobType);
- }
- }
- i++;
+ BlockSet blocks = new SimpleBlockSet();
+
+ BinaryBlob.readBinaryBlob(dis, blocks, tolerant);
+
+ Vector myInserters = new Vector();
+ Iterator i = blocks.keys().iterator();
+
+ int x=0;
+ while(i.hasNext()) {
+ Key key = (Key) i.next();
+ KeyBlock block = blocks.get(key);
+ MySendableInsert inserter =
+ new MySendableInsert(x++, block, prioClass, getScheduler(block), clientContext);
+ myInserters.add(inserter);
}
+
inserters = (MySendableInsert[]) myInserters.toArray(new MySendableInsert[myInserters.size()]);
parent.addMustSucceedBlocks(inserters.length);
parent.notifyClients();
Copied: branches/freenet-jfk/src/freenet/client/async/BlockSet.java (from rev 14796, trunk/freenet/src/freenet/client/async/BlockSet.java)
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/BlockSet.java (rev 0)
+++ branches/freenet-jfk/src/freenet/client/async/BlockSet.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -0,0 +1,42 @@
+/* This code is part of Freenet. It is distributed under the GNU General
+ * Public License, version 2 (or at your option any later version). See
+ * http://www.gnu.org/ for further details of the GPL. */
+package freenet.client.async;
+
+import java.util.Set;
+
+import freenet.keys.ClientKey;
+import freenet.keys.ClientKeyBlock;
+import freenet.keys.Key;
+import freenet.keys.KeyBlock;
+
+/**
+ * A set of KeyBlock's.
+ * @author toad
+ */
+public interface BlockSet {
+
+ /**
+ * Get a block by its key.
+ * @param key The key of the block to get.
+ * @return A block, or null if there is no block with that key.
+ */
+ public KeyBlock get(Key key);
+
+ /**
+ * Add a block.
+ * @param block The block to add.
+ */
+ public void add(KeyBlock block);
+
+ /**
+ * Get the set of all the keys of all the blocks.
+ * @return A set of the keys of the blocks in the BlockSet. Not guaranteed to be
+ * kept up to date. Read only.
+ */
+ public Set keys();
+
+ /** Get a high level block, given a high level key */
+ public ClientKeyBlock get(ClientKey key);
+
+}
Modified: branches/freenet-jfk/src/freenet/client/async/ClientGetter.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/ClientGetter.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/ClientGetter.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -3,6 +3,7 @@
* http://www.gnu.org/ for further details of the GPL. */
package freenet.client.async;
+import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -10,14 +11,13 @@
import freenet.client.ArchiveContext;
import freenet.client.ClientMetadata;
+import freenet.client.FetchContext;
import freenet.client.FetchException;
import freenet.client.FetchResult;
-import freenet.client.FetchContext;
import freenet.client.events.SplitfileProgressEvent;
import freenet.keys.ClientKeyBlock;
import freenet.keys.FreenetURI;
import freenet.keys.Key;
-import freenet.keys.KeyBlock;
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.io.BucketTools;
@@ -41,7 +41,7 @@
/** If not null, HashSet to track keys already added for a binary blob */
final HashSet binaryBlobKeysAddedAlready;
private DataOutputStream binaryBlobStream;
-
+
/**
* Fetch a key.
* @param client
@@ -98,10 +98,10 @@
if(currentState != null && !finished) {
if(binaryBlobBucket != null) {
try {
- binaryBlobStream = new DataOutputStream(binaryBlobBucket.getOutputStream());
+ binaryBlobStream = new DataOutputStream(new BufferedOutputStream(binaryBlobBucket.getOutputStream()));
BinaryBlob.writeBinaryBlobHeader(binaryBlobStream);
} catch (IOException e) {
- onFailure(new FetchException(FetchException.BUCKET_ERROR, "Failed to open binary blob bucket"), null);
+ onFailure(new FetchException(FetchException.BUCKET_ERROR, "Failed to open binary blob bucket", e), null);
return false;
}
}
@@ -142,7 +142,13 @@
if(returnBucket != null && Logger.shouldLog(Logger.MINOR, this))
Logger.minor(this, "client.async returned data in returnBucket");
}
- client.onSuccess(result, this);
+ final FetchResult res = result;
+ ctx.executor.execute(new Runnable() {
+ public void run() {
+ client.onSuccess(res, ClientGetter.this);
+ }
+ }, "ClientGetter onSuccess callback");
+
}
public void onFailure(FetchException e, ClientGetState state) {
@@ -170,13 +176,20 @@
}
synchronized(this) {
finished = true;
+ currentState = null;
}
if(e.errorCodes != null && e.errorCodes.isOneCodeOnly())
e = new FetchException(e.errorCodes.getFirstCode(), e);
if(e.mode == FetchException.DATA_NOT_FOUND && super.successfulBlocks > 0)
e = new FetchException(e, FetchException.ALL_DATA_NOT_FOUND);
Logger.minor(this, "onFailure("+e+", "+state+") on "+this+" for "+uri, e);
- client.onFailure(e, this);
+ final FetchException e1 = e;
+ ctx.executor.execute(new Runnable() {
+ public void run() {
+ client.onFailure(e1, ClientGetter.this);
+ }
+ }, "ClientGetter onFailure callback");
+
return;
}
}
@@ -267,8 +280,9 @@
* called onFailure() with an appropriate error.
*/
private boolean closeBinaryBlobStream() {
- if(binaryBlobBucket == null) return true;
+ if(binaryBlobKeysAddedAlready == null) return true;
synchronized(binaryBlobKeysAddedAlready) {
+ if(binaryBlobStream == null) return true;
try {
BinaryBlob.writeEndBlob(binaryBlobStream);
binaryBlobStream.close();
Modified: branches/freenet-jfk/src/freenet/client/async/ClientRequestScheduler.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/ClientRequestScheduler.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/ClientRequestScheduler.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -14,6 +14,8 @@
import freenet.crypt.RandomSource;
import freenet.keys.ClientKey;
import freenet.keys.ClientKeyBlock;
+import freenet.keys.Key;
+import freenet.keys.KeyBlock;
import freenet.keys.KeyVerifyException;
import freenet.node.LowLevelGetException;
import freenet.node.Node;
@@ -24,8 +26,8 @@
import freenet.node.SendableRequest;
import freenet.support.Logger;
import freenet.support.RandomGrabArray;
+import freenet.support.SectoredRandomGrabArrayWithInt;
import freenet.support.SectoredRandomGrabArrayWithObject;
-import freenet.support.SectoredRandomGrabArrayWithInt;
import freenet.support.SortedVectorByNumber;
import freenet.support.api.StringCallback;
@@ -94,6 +96,11 @@
public final String name;
private final LinkedList /* <WeakReference <RandomGrabArray> > */ recentSuccesses = new LinkedList();
+ /** All pending gets by key. Used to automatically satisfy pending requests when either the key is fetched by
+ * an overlapping request, or it is fetched by a request from another node. Operations on this are synchronized on
+ * itself. */
+ private final HashMap /* <Key, SendableGet[]> */ pendingKeys;
+
public static final String PRIORITY_NONE = "NONE";
public static final String PRIORITY_SOFT = "SOFT";
public static final String PRIORITY_HARD = "HARD";
@@ -159,6 +166,10 @@
this.isSSKScheduler = forSSKs;
priorities = new SortedVectorByNumber[RequestStarter.NUMBER_OF_PRIORITY_CLASSES];
allRequestsByClientRequest = new HashMap();
+ if(forInserts)
+ pendingKeys = null;
+ else
+ pendingKeys = new HashMap();
this.name = name;
sc.register(name+"_priority_policy", PRIORITY_HARD, name.hashCode(), true, false,
@@ -190,15 +201,22 @@
int[] keyTokens = getter.allKeys();
for(int i=0;i<keyTokens.length;i++) {
int tok = keyTokens[i];
- ClientKeyBlock block;
+ ClientKeyBlock block = null;
try {
ClientKey key = getter.getKey(tok);
if(key == null) {
if(logMINOR)
Logger.minor(this, "No key for "+tok+" for "+getter+" - already finished?");
continue;
- } else
- block = node.fetchKey(key, getter.dontCache());
+ } else {
+ if(getter.getContext().blocks != null)
+ block = getter.getContext().blocks.get(key);
+ if(block == null)
+ block = node.fetchKey(key, getter.dontCache());
+ if(block == null) {
+ addPendingKey(key, getter);
+ }
+ }
} catch (KeyVerifyException e) {
// Verify exception, probably bogus at source;
// verifies at low-level, but not at decode.
@@ -223,6 +241,36 @@
}
}
+ private void addPendingKey(ClientKey key, SendableGet getter) {
+ Key nodeKey = key.getNodeKey();
+ synchronized(pendingKeys) {
+ Object o = pendingKeys.get(nodeKey);
+ if(o == null) {
+ pendingKeys.put(nodeKey, getter);
+ } else if(o instanceof SendableGet) {
+ SendableGet oldGet = (SendableGet) o;
+ if(oldGet != getter) {
+ pendingKeys.put(nodeKey, new SendableGet[] { oldGet, getter });
+ }
+ } else {
+ SendableGet[] gets = (SendableGet[]) o;
+ boolean found = false;
+ for(int j=0;j<gets.length;j++) {
+ if(gets[j] == getter) {
+ found = true;
+ break;
+ }
+ }
+ if(!found) {
+ SendableGet[] newGets = new SendableGet[gets.length+1];
+ System.arraycopy(gets, 0, newGets, 0, gets.length);
+ newGets[gets.length] = getter;
+ pendingKeys.put(nodeKey, newGets);
+ }
+ }
+ }
+ }
+
private synchronized void innerRegister(SendableRequest req) {
if(logMINOR) Logger.minor(this, "Still registering "+req+" at prio "+req.getPriorityClass()+" retry "+req.getRetryCount());
addToGrabArray(req.getPriorityClass(), req.getRetryCount(), req.getClient(), req.getClientRequest(), req);
@@ -384,6 +432,9 @@
allRequestsByClientRequest.remove(cr);
if(logMINOR) Logger.minor(this, "Removed from HashSet for "+cr+" which now has "+v.size()+" elements");
}
+ if(!isInsertScheduler) {
+ removePendingKeys((SendableGet) req, true);
+ }
}
if(logMINOR) Logger.minor(this, "removeFirst() returning "+req);
return req;
@@ -393,6 +444,71 @@
return null;
}
+ public void removePendingKey(SendableGet getter, boolean complain, Key key) {
+ synchronized(pendingKeys) {
+ Object o = pendingKeys.get(key);
+ if(o == null) {
+ if(complain)
+ Logger.normal(this, "Not found: "+getter+" for "+key+" removing (no such key)");
+ } else if(o instanceof SendableGet) {
+ SendableGet oldGet = (SendableGet) o;
+ if(oldGet != getter) {
+ if(complain)
+ Logger.normal(this, "Not found: "+getter+" for "+key+" removing (1 getter)");
+ } else {
+ pendingKeys.remove(key);
+ }
+ } else {
+ SendableGet[] gets = (SendableGet[]) o;
+ SendableGet[] newGets = new SendableGet[gets.length-1];
+ boolean found = false;
+ int x = 0;
+ for(int j=0;j<gets.length;j++) {
+ if(j > newGets.length) {
+ if(!found) {
+ if(complain)
+ Logger.normal(this, "Not found: "+getter+" for "+key+" removing ("+gets.length+" getters)");
+ return; // not here
+ }
+ if(gets[j] == getter || gets[j] == null || gets[j].isCancelled()) continue;
+ newGets[x++] = gets[j];
+ }
+ }
+ if(x != gets.length-1) {
+ SendableGet[] newNewGets = new SendableGet[x];
+ System.arraycopy(newGets, 0, newNewGets, 0, x);
+ newGets = newNewGets;
+ }
+ if(newGets.length == 0) {
+ pendingKeys.remove(key);
+ } else if(newGets.length == 1) {
+ pendingKeys.put(key, newGets[0]);
+ } else {
+ pendingKeys.put(key, newGets);
+ }
+ }
+ }
+ }
+
+ /**
+ * Remove a SendableGet from the list of getters we maintain for each key, indicating that we are no longer interested
+ * in that key.
+ * @param getter
+ * @param complain
+ */
+ public void removePendingKeys(SendableGet getter, boolean complain) {
+ int[] keyTokens = getter.allKeys();
+ for(int i=0;i<keyTokens.length;i++) {
+ int tok = keyTokens[i];
+ ClientKey ckey = getter.getKey(tok);
+ if(ckey == null) {
+ Logger.error(this, "Key "+tok+" is null for "+getter);
+ continue;
+ }
+ removePendingKey(getter, complain, ckey.getNodeKey());
+ }
+ }
+
public void reregisterAll(ClientRequester request) {
SendableRequest[] reqs;
synchronized(this) {
@@ -403,8 +519,7 @@
for(int i=0;i<reqs.length;i++) {
SendableRequest req = reqs[i];
- RandomGrabArray array = req.getParentGrabArray();
- if(array != null) array.remove(req);
+ req.unregister();
innerRegister(req);
}
synchronized(starter) {
@@ -425,4 +540,34 @@
recentSuccesses.removeLast();
}
}
+
+ public void tripPendingKey(final KeyBlock block) {
+ final Key key = block.getKey();
+ final SendableGet[] gets;
+ Object o;
+ synchronized(pendingKeys) {
+ o = pendingKeys.get(key);
+ }
+ if(o == null) return;
+ if(o instanceof SendableGet) {
+ gets = new SendableGet[] { (SendableGet) o };
+ } else {
+ gets = (SendableGet[]) o;
+ }
+ if(gets == null) return;
+ Runnable r = new Runnable() {
+ public void run() {
+ for(int i=0;i<gets.length;i++) {
+ gets[i].onGotKey(key, block);
+ }
+ }
+ };
+ node.getTicker().queueTimedJob(r, 0); // FIXME ideally these would be completed on a single thread; when we have 1.5, use a dedicated non-parallel Executor
+ }
+
+ public boolean anyWantKey(Key key) {
+ synchronized(pendingKeys) {
+ return pendingKeys.get(key) != null;
+ }
+ }
}
Copied: branches/freenet-jfk/src/freenet/client/async/SimpleBlockSet.java (from rev 14796, trunk/freenet/src/freenet/client/async/SimpleBlockSet.java)
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/SimpleBlockSet.java (rev 0)
+++ branches/freenet-jfk/src/freenet/client/async/SimpleBlockSet.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -0,0 +1,45 @@
+package freenet.client.async;
+
+import java.util.HashMap;
+import java.util.Set;
+
+import freenet.keys.ClientKey;
+import freenet.keys.ClientKeyBlock;
+import freenet.keys.Key;
+import freenet.keys.KeyBlock;
+import freenet.keys.KeyVerifyException;
+import freenet.support.Logger;
+
+/**
+ * Simple BlockSet implementation, keeps all keys in RAM.
+ *
+ * @author toad
+ */
+public class SimpleBlockSet implements BlockSet {
+
+ private final HashMap blocksByKey = new HashMap();
+
+ public synchronized void add(KeyBlock block) {
+ blocksByKey.put(block.getKey(), block);
+ }
+
+ public synchronized KeyBlock get(Key key) {
+ return (KeyBlock) blocksByKey.get(key);
+ }
+
+ public synchronized Set keys() {
+ return blocksByKey.keySet();
+ }
+
+ public ClientKeyBlock get(ClientKey key) {
+ KeyBlock block = get(key.getNodeKey());
+ if(block == null) return null;
+ try {
+ return Key.createKeyBlock(key, block);
+ } catch (KeyVerifyException e) {
+ Logger.error(this, "Caught decoding block with "+key+" : "+e, e);
+ return null;
+ }
+ }
+
+}
Modified: branches/freenet-jfk/src/freenet/client/async/SimpleManifestPutter.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/SimpleManifestPutter.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/SimpleManifestPutter.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -1,5 +1,6 @@
package freenet.client.async;
+import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.HashMap;
@@ -404,7 +405,7 @@
// Only the *decoding* is generic at present.
Bucket zipBucket = ctx.bf.makeBucket(-1);
- OutputStream os = zipBucket.getOutputStream();
+ OutputStream os = new BufferedOutputStream(zipBucket.getOutputStream());
ZipOutputStream zos = new ZipOutputStream(os);
ZipEntry ze;
Modified: branches/freenet-jfk/src/freenet/client/async/SimpleSingleFileFetcher.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/SimpleSingleFileFetcher.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/SimpleSingleFileFetcher.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -47,6 +47,9 @@
case LowLevelGetException.DATA_NOT_FOUND_IN_STORE:
onFailure(new FetchException(FetchException.DATA_NOT_FOUND));
return;
+ case LowLevelGetException.RECENTLY_FAILED:
+ onFailure(new FetchException(FetchException.RECENTLY_FAILED));
+ return;
case LowLevelGetException.DECODE_FAILED:
onFailure(new FetchException(FetchException.BLOCK_DECODE_ERROR));
return;
@@ -95,6 +98,7 @@
}
}
// :(
+ unregister();
if(e.isFatal() || forceFatal)
parent.fatallyFailedBlock();
else
@@ -104,6 +108,7 @@
/** Will be overridden by SingleFileFetcher */
protected void onSuccess(FetchResult data) {
+ unregister();
if(parent.isCancelled()) {
data.asBucket().free();
onFailure(new FetchException(FetchException.CANCELLED));
Modified: branches/freenet-jfk/src/freenet/client/async/SingleBlockInserter.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/SingleBlockInserter.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/SingleBlockInserter.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -21,7 +21,6 @@
import freenet.node.RequestScheduler;
import freenet.node.SendableInsert;
import freenet.support.Logger;
-import freenet.support.RandomGrabArray;
import freenet.support.SimpleFieldSet;
import freenet.support.api.Bucket;
@@ -163,7 +162,7 @@
Logger.error(this, "Unknown LowLevelPutException code: "+e.code);
errors.inc(InsertException.INTERNAL_ERROR);
}
- if(e.code == LowLevelPutException.ROUTE_NOT_FOUND) {
+ if(e.code == LowLevelPutException.ROUTE_NOT_FOUND || e.code == LowLevelPutException.ROUTE_REALLY_NOT_FOUND) {
consecutiveRNFs++;
if(logMINOR) Logger.minor(this, "Consecutive RNFs: "+consecutiveRNFs+" / "+ctx.consecutiveRNFsCountAsSuccess);
if(consecutiveRNFs == ctx.consecutiveRNFsCountAsSuccess) {
@@ -276,8 +275,7 @@
if(finished) return;
finished = true;
}
- RandomGrabArray arr = getParentGrabArray();
- if(arr != null) arr.remove(this);
+ super.unregister();
cb.onFailure(new InsertException(InsertException.CANCELLED), this);
}
Modified: branches/freenet-jfk/src/freenet/client/async/SingleFileFetcher.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/SingleFileFetcher.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/SingleFileFetcher.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -8,7 +8,9 @@
import java.util.LinkedList;
import freenet.client.ArchiveContext;
+import freenet.client.ArchiveExtractCallback;
import freenet.client.ArchiveFailureException;
+import freenet.client.ArchiveManager;
import freenet.client.ArchiveRestartException;
import freenet.client.ArchiveStoreContext;
import freenet.client.ClientMetadata;
@@ -158,23 +160,12 @@
onFailure(new FetchException(FetchException.BUCKET_ERROR, e));
return;
}
- try {
- handleMetadata();
- } catch (MetadataParseException e) {
- onFailure(new FetchException(e));
- return;
- } catch (FetchException e) {
- onFailure(e);
- return;
- } catch (ArchiveFailureException e) {
- onFailure(new FetchException(e));
- } catch (ArchiveRestartException e) {
- onFailure(new FetchException(e));
- }
+ wrapHandleMetadata(false);
}
}
protected void onSuccess(FetchResult result) {
+ unregister();
if(parent.isCancelled()) {
if(logMINOR)
Logger.minor(this, "Parent is cancelled");
@@ -227,20 +218,29 @@
}
}
- private void handleMetadata() throws FetchException, MetadataParseException, ArchiveFailureException, ArchiveRestartException {
+ /**
+ * Handle the current metadata. I.e. do something with it: transition to a splitfile, look up a manifest, etc.
+ * LOCKING: Synchronized as it changes so many variables; if we want to write the structure to disk, we don't
+ * want this running at the same time.
+ * @throws FetchException
+ * @throws MetadataParseException
+ * @throws ArchiveFailureException
+ * @throws ArchiveRestartException
+ */
+ private synchronized void handleMetadata() throws FetchException, MetadataParseException, ArchiveFailureException, ArchiveRestartException {
while(true) {
if(metadata.isSimpleManifest()) {
if(logMINOR) Logger.minor(this, "Is simple manifest");
String name;
if(metaStrings.isEmpty())
- throw new FetchException(FetchException.NOT_ENOUGH_PATH_COMPONENTS);
+ throw new FetchException(FetchException.NOT_ENOUGH_PATH_COMPONENTS, -1, false, null, thisKey.addMetaStrings(new String[] { "" }));
else name = removeMetaString();
// Since metadata is a document, we just replace metadata here
if(logMINOR) Logger.minor(this, "Next meta-string: "+name);
if(name == null) {
metadata = metadata.getDefaultDocument();
if(metadata == null)
- throw new FetchException(FetchException.NOT_ENOUGH_PATH_COMPONENTS);
+ throw new FetchException(FetchException.NOT_ENOUGH_PATH_COMPONENTS, -1, false, null, thisKey.addMetaStrings(new String[] { "" }));
} else {
metadata = metadata.getDocument(name);
thisKey = thisKey.pushMetaString(name);
@@ -274,13 +274,35 @@
throw new FetchException(FetchException.BUCKET_ERROR, e);
}
} else {
- fetchArchive(false, archiveMetadata); // will result in this function being called again
+ fetchArchive(false, archiveMetadata, ArchiveManager.METADATA_NAME, new ArchiveExtractCallback() {
+ public void gotBucket(Bucket data) {
+ try {
+ metadata = Metadata.construct(data);
+ } catch (MetadataParseException e) {
+ // Invalid metadata
+ onFailure(new FetchException(FetchException.INVALID_METADATA, e));
+ return;
+ } catch (IOException e) {
+ // Bucket error?
+ onFailure(new FetchException(FetchException.BUCKET_ERROR, e));
+ return;
+ }
+ wrapHandleMetadata(true);
+ }
+ public void notInArchive() {
+ onFailure(new FetchException(FetchException.INTERNAL_ERROR, "No metadata in container! Cannot happen as ArchiveManager should synthesise some!"));
+ }
+ }); // will result in this function being called again
return;
}
continue;
} else if(metadata.isArchiveInternalRedirect()) {
if(logMINOR) Logger.minor(this, "Is archive-internal redirect");
clientMetadata.mergeNoOverwrite(metadata.getClientMetadata());
+ if(metaStrings.isEmpty() && isFinal && clientMetadata.getMIMETypeNoParams() != null && ctx.allowedMIMETypes != null &&
+ !ctx.allowedMIMETypes.contains(clientMetadata.getMIMETypeNoParams())) {
+ throw new FetchException(FetchException.WRONG_MIME_TYPE, -1, false, clientMetadata.getMIMEType());
+ }
// Fetch it from the archive
if(ah == null)
throw new FetchException(FetchException.UNKNOWN_METADATA, "Archive redirect not in an archive manifest");
@@ -289,30 +311,56 @@
Bucket dataBucket = ah.get(filename, actx, null, recursionLevel+1, true);
if(dataBucket != null) {
if(logMINOR) Logger.minor(this, "Returning data");
- // The client may free it, which is bad, or it may hang on to it for so long that it gets
- // freed by us, which is also bad.
- // So copy it.
- // FIXME this is stupid, reconsider how we determine when to free buckets; refcounts maybe?
- Bucket out;
+ final Bucket out;
try {
- if(returnBucket != null)
+ // Data will not be freed until client is finished with it.
+ if(returnBucket != null) {
out = returnBucket;
- else
- out = ctx.bucketFactory.makeBucket(dataBucket.size());
- BucketTools.copy(dataBucket, out);
+ BucketTools.copy(dataBucket, out);
+ dataBucket.free();
+ } else {
+ out = dataBucket;
+ }
} catch (IOException e) {
- onFailure(new FetchException(FetchException.BUCKET_ERROR));
- return;
+ throw new FetchException(FetchException.BUCKET_ERROR);
}
// Return the data
- onSuccess(new FetchResult(this.clientMetadata, out));
+ ctx.executor.execute(new Runnable() {
+ public void run() {
+ onSuccess(new FetchResult(clientMetadata, out));
+ }
+ }, "SingleFileFetcher onSuccess callback for "+this);
+
return;
} else {
if(logMINOR) Logger.minor(this, "Fetching archive (thisKey="+thisKey+ ')');
// Metadata cannot contain pointers to files which don't exist.
// We enforce this in ArchiveHandler.
// Therefore, the archive needs to be fetched.
- fetchArchive(true, archiveMetadata);
+ fetchArchive(true, archiveMetadata, filename, new ArchiveExtractCallback() {
+ public void gotBucket(Bucket data) {
+ if(logMINOR) Logger.minor(this, "Returning data");
+ Bucket out;
+ try {
+ // Data will not be freed until client is finished with it.
+ if(returnBucket != null) {
+ out = returnBucket;
+ BucketTools.copy(data, out);
+ data.free();
+ } else {
+ out = data;
+ }
+ } catch (IOException e) {
+ onFailure(new FetchException(FetchException.BUCKET_ERROR));
+ return;
+ }
+ // Return the data
+ onSuccess(new FetchResult(clientMetadata, out));
+ }
+ public void notInArchive() {
+ onFailure(new FetchException(FetchException.NOT_IN_ARCHIVE));
+ }
+ });
// Will call back into this function when it has been fetched.
return;
}
@@ -321,14 +369,31 @@
// Fetch on a second SingleFileFetcher, like with archives.
Metadata newMeta = (Metadata) metadata.clone();
newMeta.setSimpleRedirect();
- SingleFileFetcher f = new SingleFileFetcher(this, newMeta, new MultiLevelMetadataCallback(), ctx);
- f.handleMetadata();
+ final SingleFileFetcher f = new SingleFileFetcher(this, newMeta, new MultiLevelMetadataCallback(), ctx);
+ ctx.ticker.queueTimedJob(new Runnable() {
+ public void run() {
+ f.wrapHandleMetadata(true);
+ }
+ }, 0);
return;
} else if(metadata.isSingleFileRedirect()) {
if(logMINOR) Logger.minor(this, "Is single-file redirect");
clientMetadata.mergeNoOverwrite(metadata.getClientMetadata()); // even splitfiles can have mime types!
- // FIXME implement implicit archive support
+
+ String mimeType = clientMetadata.getMIMETypeNoParams();
+ if(mimeType != null && ArchiveManager.isUsableArchiveType(mimeType) && metaStrings.size() > 0) {
+ // Looks like an implicit archive, handle as such
+ metadata.setArchiveManifest();
+ // Pick up MIME type from inside archive
+ clientMetadata.clear();
+ continue;
+ }
+ if(metaStrings.isEmpty() && isFinal && mimeType != null && ctx.allowedMIMETypes != null &&
+ !ctx.allowedMIMETypes.contains(mimeType)) {
+ throw new FetchException(FetchException.WRONG_MIME_TYPE, -1, false, clientMetadata.getMIMEType());
+ }
+
// Simple redirect
// Just create a new SingleFileFetcher
// Which will then fetch the target URI, and call the rcd.success
@@ -358,7 +423,7 @@
}
// **FIXME** Is key in the call to SingleFileFetcher here supposed to be this.key or the same key used in the try block above? MultiLevelMetadataCallback.onSuccess() below uses this.key, thus the question
- SingleFileFetcher f = new SingleFileFetcher(parent, rcb, clientMetadata, key, metaStrings, this.uri, addedMetaStrings, ctx, actx, ah, maxRetries, recursionLevel, false, token, true, returnBucket, isFinal);
+ final SingleFileFetcher f = new SingleFileFetcher(parent, rcb, clientMetadata, key, metaStrings, this.uri, addedMetaStrings, ctx, actx, ah, maxRetries, recursionLevel, false, token, true, returnBucket, isFinal);
if((key instanceof ClientCHK) && !((ClientCHK)key).isMetadata())
rcb.onBlockSetFinished(this);
if(metadata.isCompressed()) {
@@ -366,15 +431,32 @@
f.addDecompressor(codec);
}
parent.onTransition(this, f);
- f.schedule();
+ ctx.executor.execute(new Runnable() {
+ public void run() {
+ f.schedule();
+ }
+ }, "Schedule");
// All done! No longer our problem!
return;
} else if(metadata.isSplitfile()) {
if(logMINOR) Logger.minor(this, "Fetching splitfile");
- // FIXME implicit archive support
clientMetadata.mergeNoOverwrite(metadata.getClientMetadata()); // even splitfiles can have mime types!
+ String mimeType = clientMetadata.getMIMETypeNoParams();
+ if(mimeType != null && ArchiveManager.isUsableArchiveType(mimeType) && metaStrings.size() > 0) {
+ // Looks like an implicit archive, handle as such
+ metadata.setArchiveManifest();
+ // Pick up MIME type from inside archive
+ clientMetadata.clear();
+ continue;
+ }
+
+ if(metaStrings.isEmpty() && isFinal && mimeType != null && ctx.allowedMIMETypes != null &&
+ !ctx.allowedMIMETypes.contains(mimeType)) {
+ throw new FetchException(FetchException.WRONG_MIME_TYPE, metadata.uncompressedDataLength(), false, clientMetadata.getMIMEType());
+ }
+
// Splitfile (possibly compressed)
if(metadata.isCompressed()) {
@@ -410,8 +492,7 @@
if((len > ctx.maxOutputLength) ||
(len > ctx.maxTempLength)) {
- onFailure(new FetchException(FetchException.TOO_BIG, len, isFinal && decompressors.size() <= (metadata.isCompressed() ? 1 : 0), clientMetadata.getMIMEType()));
- return;
+ throw new FetchException(FetchException.TOO_BIG, len, isFinal && decompressors.size() <= (metadata.isCompressed() ? 1 : 0), clientMetadata.getMIMEType());
}
SplitFileFetcher sf = new SplitFileFetcher(metadata, rcb, parent, ctx,
@@ -440,7 +521,7 @@
decompressors.addLast(codec);
}
- private void fetchArchive(boolean forData, Metadata meta) throws FetchException, MetadataParseException, ArchiveFailureException, ArchiveRestartException {
+ private void fetchArchive(boolean forData, Metadata meta, String element, ArchiveExtractCallback callback) throws FetchException, MetadataParseException, ArchiveFailureException, ArchiveRestartException {
if(logMINOR) Logger.minor(this, "fetchArchive()");
// Fetch the archive
// How?
@@ -450,42 +531,61 @@
// reschedules us.
Metadata newMeta = (Metadata) meta.clone();
newMeta.setSimpleRedirect();
- SingleFileFetcher f;
- f = new SingleFileFetcher(this, newMeta, new ArchiveFetcherCallback(forData), new FetchContext(ctx, FetchContext.SET_RETURN_ARCHIVES, true));
- f.handleMetadata();
- // When it is done (if successful), the ArchiveCallback will re-call this function.
- // Which will then discover that the metadata *is* available.
- // And will also discover that the data is available, and will complete.
+ final SingleFileFetcher f;
+ f = new SingleFileFetcher(this, newMeta, new ArchiveFetcherCallback(forData, element, callback), new FetchContext(ctx, FetchContext.SET_RETURN_ARCHIVES, true));
+ ctx.ticker.queueTimedJob(new Runnable() {
+ public void run() {
+ // Fetch the archive. The archive fetcher callback will unpack it, and either call the element
+ // callback, or just go back around handleMetadata() on this, which will see that the data is now
+ // available.
+ f.wrapHandleMetadata(true);
+ }
+ }, 0);
}
+ /**
+ * Call handleMetadata(), and deal with any resulting exceptions
+ */
+ private void wrapHandleMetadata(boolean notFinalizedSize) {
+ try {
+ handleMetadata();
+ } catch (MetadataParseException e) {
+ onFailure(new FetchException(e));
+ } catch (FetchException e) {
+ if(notFinalizedSize)
+ e.setNotFinalizedSize();
+ onFailure(e);
+ } catch (ArchiveFailureException e) {
+ onFailure(new FetchException(e));
+ } catch (ArchiveRestartException e) {
+ onFailure(new FetchException(e));
+ }
+ }
+
class ArchiveFetcherCallback implements GetCompletionCallback {
private final boolean wasFetchingFinalData;
+ private final String element;
+ private final ArchiveExtractCallback callback;
- ArchiveFetcherCallback(boolean wasFetchingFinalData) {
+ ArchiveFetcherCallback(boolean wasFetchingFinalData, String element, ArchiveExtractCallback cb) {
this.wasFetchingFinalData = wasFetchingFinalData;
+ this.element = element;
+ this.callback = cb;
}
public void onSuccess(FetchResult result, ClientGetState state) {
try {
- ah.extractToCache(result.asBucket(), actx);
+ ah.extractToCache(result.asBucket(), actx, element, callback);
} catch (ArchiveFailureException e) {
SingleFileFetcher.this.onFailure(new FetchException(e));
+ return;
} catch (ArchiveRestartException e) {
SingleFileFetcher.this.onFailure(new FetchException(e));
+ return;
}
- try {
- handleMetadata();
- } catch (MetadataParseException e) {
- SingleFileFetcher.this.onFailure(new FetchException(e));
- } catch (FetchException e) {
- e.setNotFinalizedSize();
- SingleFileFetcher.this.onFailure(e);
- } catch (ArchiveFailureException e) {
- SingleFileFetcher.this.onFailure(new FetchException(e));
- } catch (ArchiveRestartException e) {
- SingleFileFetcher.this.onFailure(new FetchException(e));
- }
+ if(callback != null) return;
+ wrapHandleMetadata(true);
}
public void onFailure(FetchException e, ClientGetState state) {
@@ -510,22 +610,15 @@
public void onSuccess(FetchResult result, ClientGetState state) {
try {
metadata = Metadata.construct(result.asBucket());
- handleMetadata();
} catch (MetadataParseException e) {
- SingleFileFetcher.this.onFailure(new FetchException(e));
+ SingleFileFetcher.this.onFailure(new FetchException(FetchException.INVALID_METADATA, e));
return;
} catch (IOException e) {
// Bucket error?
SingleFileFetcher.this.onFailure(new FetchException(FetchException.BUCKET_ERROR, e));
return;
- } catch (FetchException e) {
- e.setNotFinalizedSize();
- onFailure(e, SingleFileFetcher.this);
- } catch (ArchiveFailureException e) {
- onFailure(new FetchException(FetchException.ARCHIVE_FAILURE), SingleFileFetcher.this);
- } catch (ArchiveRestartException e) {
- onFailure(new FetchException(FetchException.ARCHIVE_RESTART), SingleFileFetcher.this);
}
+ wrapHandleMetadata(true);
}
public void onFailure(FetchException e, ClientGetState state) {
Modified: branches/freenet-jfk/src/freenet/client/async/SingleFileInserter.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/SingleFileInserter.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/SingleFileInserter.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -106,10 +106,7 @@
if(data.size() > COMPRESS_OFF_THREAD_LIMIT) {
// Run off thread
OffThreadCompressor otc = new OffThreadCompressor();
- Thread t = new Thread(otc, "Compressor for "+this);
- if(logMINOR) Logger.minor(this, "Compressing off-thread: "+t);
- t.setDaemon(true);
- t.start();
+ ctx.executor.execute(otc, "Compressor for "+this);
} else {
tryCompress();
}
@@ -214,7 +211,7 @@
boolean fitsInOneBlockAsIs = bestCodec == null ? compressedDataSize < blockSize : compressedDataSize < oneBlockCompressedSize;
boolean fitsInOneCHK = bestCodec == null ? compressedDataSize < CHKBlock.DATA_LENGTH : compressedDataSize < CHKBlock.MAX_COMPRESSED_DATA_LENGTH;
- if(block.getData().size() > Integer.MAX_VALUE)
+ if((fitsInOneBlockAsIs || fitsInOneCHK) && block.getData().size() > Integer.MAX_VALUE)
throw new InsertException(InsertException.INTERNAL_ERROR, "2GB+ should not encode to one block!", null);
boolean noMetadata = ((block.clientMetadata == null) || block.clientMetadata.isTrivial()) && targetFilename == null;
Modified: branches/freenet-jfk/src/freenet/client/async/SplitFileFetcher.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/SplitFileFetcher.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/SplitFileFetcher.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -46,10 +46,6 @@
final int segmentCount;
/** The detailed information on each segment */
final SplitFileFetcherSegment[] segments;
- /** The splitfile data blocks. */
- final ClientCHK[] splitfileDataBlocks;
- /** The splitfile check blocks. */
- final ClientCHK[] splitfileCheckBlocks;
/** Maximum temporary length */
final long maxTempLength;
/** Have all segments finished? Access synchronized. */
@@ -77,8 +73,8 @@
throw new FetchException(FetchException.CANCELLED);
overrideLength = metadata.dataLength();
this.splitfileType = metadata.getSplitfileType();
- splitfileDataBlocks = metadata.getSplitfileDataKeys();
- splitfileCheckBlocks = metadata.getSplitfileCheckKeys();
+ ClientCHK[] splitfileDataBlocks = metadata.getSplitfileDataKeys();
+ ClientCHK[] splitfileCheckBlocks = metadata.getSplitfileCheckKeys();
for(int i=0;i<splitfileDataBlocks.length;i++)
if(splitfileDataBlocks[i] == null) throw new MetadataParseException("Null: data block "+i+" of "+splitfileDataBlocks.length);
for(int i=0;i<splitfileCheckBlocks.length;i++)
@@ -283,13 +279,11 @@
}
public void scheduleOffThread() {
- Thread t = new Thread(new Runnable() {
+ fetchContext.executor.execute(new Runnable() {
public void run() {
schedule();
}
}, "Splitfile scheduler thread for "+this);
- t.setDaemon(true);
- t.start();
}
}
Modified: branches/freenet-jfk/src/freenet/client/async/SplitFileFetcherSegment.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/SplitFileFetcherSegment.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/SplitFileFetcherSegment.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -22,6 +22,7 @@
import freenet.keys.ClientCHK;
import freenet.keys.ClientCHKBlock;
import freenet.keys.ClientKeyBlock;
+import freenet.keys.NodeCHK;
import freenet.support.Logger;
import freenet.support.api.Bucket;
import freenet.support.io.BucketTools;
@@ -190,13 +191,6 @@
// Now decode
if(logMINOR) Logger.minor(this, "Decoding "+SplitFileFetcherSegment.this);
- boolean[] dataBlocksSucceeded = new boolean[dataBuckets.length];
- boolean[] checkBlocksSucceeded = new boolean[checkBuckets.length];
- for(int i=0;i<dataBuckets.length;i++)
- dataBlocksSucceeded[i] = dataBuckets[i].data != null;
- for(int i=0;i<checkBuckets.length;i++)
- checkBlocksSucceeded[i] = checkBuckets[i].data != null;
-
codec = FECCodec.getCodec(splitfileType, dataKeys.length, checkKeys.length);
if(splitfileType != Metadata.SPLITFILE_NONREDUNDANT) {
@@ -210,7 +204,12 @@
if(isCollectingBinaryBlob()) {
for(int i=0;i<dataBuckets.length;i++) {
Bucket data = dataBuckets[i].getData();
- maybeAddToBinaryBlob(data, i, false);
+ try {
+ maybeAddToBinaryBlob(data, i, false);
+ } catch (FetchException e) {
+ fail(e);
+ return;
+ }
}
}
decodedData = fetchContext.bucketFactory.makeBucket(-1);
@@ -268,7 +267,12 @@
for(int i=0;i<checkBuckets.length;i++) {
boolean heal = false;
Bucket data = checkBuckets[i].getData();
- maybeAddToBinaryBlob(data, i, true);
+ try {
+ maybeAddToBinaryBlob(data, i, true);
+ } catch (FetchException e) {
+ fail(e);
+ return;
+ }
if(checkRetries[i] > 0)
heal = true;
if(heal) {
@@ -291,7 +295,7 @@
} else return false;
}
- private void maybeAddToBinaryBlob(Bucket data, int i, boolean check) {
+ private void maybeAddToBinaryBlob(Bucket data, int i, boolean check) throws FetchException {
if(parentFetcher.parent instanceof ClientGetter) {
ClientGetter getter = (ClientGetter) (parentFetcher.parent);
if(getter.collectingBinaryBlob()) {
@@ -301,11 +305,9 @@
getter.addKeyToBinaryBlob(block);
} catch (CHKEncodeException e) {
Logger.error(this, "Failed to encode (collecting binary blob) "+(check?"check":"data")+" block "+i+": "+e, e);
- fail(new FetchException(FetchException.INTERNAL_ERROR, "Failed to encode for binary blob: "+e));
- return;
+ throw new FetchException(FetchException.INTERNAL_ERROR, "Failed to encode for binary blob: "+e);
} catch (IOException e) {
- fail(new FetchException(FetchException.BUCKET_ERROR, "Failed to encode for binary blob: "+e));
- return;
+ throw new FetchException(FetchException.BUCKET_ERROR, "Failed to encode for binary blob: "+e);
}
}
}
@@ -321,6 +323,9 @@
logMINOR = Logger.shouldLog(Logger.MINOR, this);
if(logMINOR) Logger.minor(this, "Permanently failed block: "+blockNo+" on "+this+" : "+e, e);
boolean allFailed;
+ // Since we can't keep the key, we need to unregister for it at this point to avoid a memory leak
+ NodeCHK key = getBlockNodeKey(blockNo);
+ if(key != null) seg.unregisterKey(key);
synchronized(this) {
if(isFinishing()) return; // this failure is now irrelevant, and cleanup will occur on the decoder thread
if(blockNo < dataKeys.length) {
@@ -345,11 +350,13 @@
failedBlocks++;
parentFetcher.parent.failedBlock();
}
- allFailed = failedBlocks + fatallyFailedBlocks <= (dataKeys.length + checkKeys.length - minFetched);
+ // Once it is no longer possible to have a successful fetch, fail...
+ allFailed = failedBlocks + fatallyFailedBlocks > (dataKeys.length + checkKeys.length - minFetched);
}
if(allFailed)
fail(new FetchException(FetchException.SPLITFILE_ERROR, errors));
- seg.possiblyRemoveFromParent();
+ else
+ seg.possiblyRemoveFromParent();
}
/** A request has failed non-fatally, so the block may be retried */
@@ -418,6 +425,7 @@
checkBuckets[i] = null;
}
}
+ removeSubSegments();
parentFetcher.segmentFinished(this);
}
@@ -449,11 +457,19 @@
}
public ClientCHK getBlockKey(int blockNum) {
- if(blockNum < dataKeys.length)
+ if(blockNum < 0) return null;
+ else if(blockNum < dataKeys.length)
return dataKeys[blockNum];
- else
+ else if(blockNum < dataKeys.length + checkKeys.length)
return checkKeys[blockNum - dataKeys.length];
+ else return null;
}
+
+ public NodeCHK getBlockNodeKey(int blockNum) {
+ ClientCHK key = getBlockKey(blockNum);
+ if(key != null) return key.getNodeCHK();
+ else return null;
+ }
public synchronized void removeSeg(SplitFileFetcherSubSegment segment) {
for(int i=0;i<subSegments.size();i++) {
@@ -464,4 +480,11 @@
}
}
+ private void removeSubSegments() {
+ for(int i=0;i<subSegments.size();i++) {
+ SplitFileFetcherSubSegment seg = (SplitFileFetcherSubSegment) subSegments.get(i);
+ seg.kill();
+ }
+ }
+
}
Modified: branches/freenet-jfk/src/freenet/client/async/SplitFileFetcherSubSegment.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/SplitFileFetcherSubSegment.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/SplitFileFetcherSubSegment.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -7,7 +7,10 @@
import freenet.client.FetchException;
import freenet.keys.ClientKey;
import freenet.keys.ClientKeyBlock;
+import freenet.keys.Key;
+import freenet.keys.KeyBlock;
import freenet.keys.KeyDecodeException;
+import freenet.keys.KeyVerifyException;
import freenet.keys.TooBigException;
import freenet.node.LowLevelGetException;
import freenet.node.SendableGet;
@@ -96,6 +99,9 @@
case LowLevelGetException.DATA_NOT_FOUND_IN_STORE:
onFailure(new FetchException(FetchException.DATA_NOT_FOUND), token);
return;
+ case LowLevelGetException.RECENTLY_FAILED:
+ onFailure(new FetchException(FetchException.RECENTLY_FAILED), token);
+ return;
case LowLevelGetException.DECODE_FAILED:
onFailure(new FetchException(FetchException.BLOCK_DECODE_ERROR), token);
return;
@@ -253,9 +259,46 @@
return super.toString()+":"+retryCount+"/"+segment+'('+blockNums.size()+')';
}
- public synchronized void possiblyRemoveFromParent() {
- if(blockNums.isEmpty())
- segment.removeSeg(this);
+ public void possiblyRemoveFromParent() {
+ synchronized(this) {
+ if(!blockNums.isEmpty()) return;
+ }
+ segment.removeSeg(this);
+ unregister();
}
+
+ public void onGotKey(Key key, KeyBlock block) {
+ int blockNum = -1;
+ ClientKey ckey = null;
+ synchronized(this) {
+ for(int i=0;i<blockNums.size();i++) {
+ int num = ((Integer)blockNums.get(i)).intValue();
+ ckey = segment.getBlockKey(num);
+ if(ckey == null) return; // Already got this key
+ Key k = ckey.getNodeKey();
+ if(k.equals(key)) {
+ blockNum = num;
+ blockNums.remove(i);
+ break;
+ }
+ }
+ }
+ if(blockNum == -1) return;
+ try {
+ onSuccess(Key.createKeyBlock(ckey, block), false, blockNum);
+ } catch (KeyVerifyException e) {
+ // FIXME if we ever abolish the direct route, this must be turned into an onFailure().
+ Logger.error(this, "Failed to parse in onGotKey("+key+","+block+") - believed to be "+ckey+" (block #"+blockNum+")");
+ }
+ }
+ public void kill() {
+ // Do unregister() first so can get and unregister each key and avoid a memory leak
+ unregister();
+ synchronized(this) {
+ blockNums.clear();
+ }
+ segment.removeSeg(this);
+ }
+
}
Modified: branches/freenet-jfk/src/freenet/client/async/USKChecker.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/USKChecker.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/USKChecker.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -26,6 +26,7 @@
}
public void onSuccess(ClientKeyBlock block, boolean fromStore, int token) {
+ unregister();
cb.onSuccess((ClientSSKBlock)block);
}
@@ -42,6 +43,7 @@
break;
case LowLevelGetException.DATA_NOT_FOUND:
case LowLevelGetException.DATA_NOT_FOUND_IN_STORE:
+ case LowLevelGetException.RECENTLY_FAILED:
dnfs++;
canRetry = true;
break;
@@ -61,7 +63,7 @@
if(canRetry && retry()) return;
// Ran out of retries.
-
+ unregister();
if(e.code == LowLevelGetException.CANCELLED){
cb.onCancelled();
return;
Modified: branches/freenet-jfk/src/freenet/client/async/USKFetcher.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/USKFetcher.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/USKFetcher.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -150,9 +150,10 @@
}
public void schedule() {
- if(checker == null)
- Logger.error(this, "Checker == null in schedule()", new Exception("error"));
- else
+ if(checker == null) {
+ if(logMINOR)
+ Logger.minor(this, "Checker == null in schedule() for "+this, new Exception("debug"));
+ } else
checker.schedule();
}
Modified: branches/freenet-jfk/src/freenet/client/async/USKInserter.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/USKInserter.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/USKInserter.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -69,11 +69,13 @@
* The Fetcher must be insert-mode, in other words, it must know that we want the latest edition,
* including author errors and so on.
*/
- private synchronized void scheduleFetcher() {
- if(Logger.shouldLog(Logger.MINOR, this))
- Logger.minor(this, "scheduling fetcher for "+pubUSK.getURI());
- if(finished) return;
- fetcher = ctx.uskManager.getFetcherForInsertDontSchedule(pubUSK, parent.priorityClass, this, parent.getClient());
+ private void scheduleFetcher() {
+ synchronized(this) {
+ if(Logger.shouldLog(Logger.MINOR, this))
+ Logger.minor(this, "scheduling fetcher for "+pubUSK.getURI());
+ if(finished) return;
+ fetcher = ctx.uskManager.getFetcherForInsertDontSchedule(pubUSK, parent.priorityClass, this, parent.getClient());
+ }
fetcher.schedule();
}
Modified: branches/freenet-jfk/src/freenet/client/async/USKManager.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/USKManager.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/USKManager.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -10,6 +10,7 @@
import freenet.keys.USK;
import freenet.node.NodeClientCore;
import freenet.node.RequestStarter;
+import freenet.node.Ticker;
import freenet.support.LRUQueue;
import freenet.support.Logger;
@@ -40,6 +41,8 @@
final FetchContext backgroundFetchContext;
final ClientRequestScheduler chkRequestScheduler;
final ClientRequestScheduler sskRequestScheduler;
+
+ final Ticker ticker;
public USKManager(NodeClientCore core) {
@@ -54,6 +57,7 @@
checkersByUSK = new HashMap();
backgroundFetchersByClearUSK = new HashMap();
temporaryBackgroundFetchersLRU = new LRUQueue();
+ ticker = core.getTicker();
}
/**
@@ -125,11 +129,11 @@
if(sched != null) sched.schedule();
}
- void update(USK origUSK, long number) {
+ void update(final USK origUSK, final long number) {
boolean logMINOR = Logger.shouldLog(Logger.MINOR, this);
if(logMINOR) Logger.minor(this, "Updating "+origUSK.getURI()+" : "+number);
USK clear = origUSK.clearCopy();
- USKCallback[] callbacks;
+ final USKCallback[] callbacks;
synchronized(this) {
Long l = (Long) latestVersionByClearUSK.get(clear);
if(logMINOR) Logger.minor(this, "Old value: "+l);
@@ -141,9 +145,14 @@
callbacks = (USKCallback[]) subscribersByClearUSK.get(clear);
}
if(callbacks != null) {
- USK usk = origUSK.copy(number);
- for(int i=0;i<callbacks.length;i++)
- callbacks[i].onFoundEdition(number, usk);
+ // Run off-thread, because of locking, and because client callbacks may take some time
+ ticker.queueTimedJob(new Runnable() {
+ public void run() {
+ USK usk = origUSK.copy(number);
+ for(int i=0;i<callbacks.length;i++)
+ callbacks[i].onFoundEdition(number, usk);
+ }
+ }, 0);
}
}
@@ -163,6 +172,8 @@
if(callbacks == null)
callbacks = new USKCallback[1];
else {
+ for(int i=0;i<callbacks.length;i++)
+ if(callbacks[i] == cb) return;
USKCallback[] newCallbacks = new USKCallback[callbacks.length+1];
System.arraycopy(callbacks, 0, newCallbacks, 0, callbacks.length);
callbacks = newCallbacks;
@@ -181,8 +192,14 @@
}
if(curEd > ed)
cb.onFoundEdition(curEd, origUSK.copy(curEd));
- if(sched != null)
- sched.schedule();
+ final USKFetcher fetcher = sched;
+ if(fetcher != null) {
+ ticker.queueTimedJob(new Runnable() {
+ public void run() {
+ fetcher.schedule();
+ }
+ }, 0);
+ }
}
public void unsubscribe(USK origUSK, USKCallback cb, boolean runBackgroundFetch) {
Modified: branches/freenet-jfk/src/freenet/client/async/USKProxyCompletionCallback.java
===================================================================
--- branches/freenet-jfk/src/freenet/client/async/USKProxyCompletionCallback.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/client/async/USKProxyCompletionCallback.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -5,6 +5,7 @@
import freenet.client.FetchException;
import freenet.client.FetchResult;
+import freenet.keys.FreenetURI;
import freenet.keys.USK;
public class USKProxyCompletionCallback implements GetCompletionCallback {
@@ -25,6 +26,11 @@
}
public void onFailure(FetchException e, ClientGetState state) {
+ FreenetURI uri = e.newURI;
+ if(uri != null) {
+ uri = usk.turnMySSKIntoUSK(uri);
+ e = new FetchException(e, uri);
+ }
cb.onFailure(e, state);
}
Modified: branches/freenet-jfk/src/freenet/clients/http/BookmarkEditorToadlet.java
===================================================================
--- branches/freenet-jfk/src/freenet/clients/http/BookmarkEditorToadlet.java 2007-08-21 19:57:05 UTC (rev 14827)
+++ branches/freenet-jfk/src/freenet/clients/http/BookmarkEditorToadlet.java 2007-08-21 20:26:59 UTC (rev 14828)
@@ -16,6 +16,9 @@
import freenet.node.NodeClientCore;
import freenet.client.HighLevelSimpleClient;
import freenet.support.HTMLNode;
+import freenet.support.URLDecoder;
+import freenet.support.URLEncodedFormatException;
+import freenet.support.URLEncoder;
import freenet.support.api.HTTPRequest;
public class BookmarkEditorToadlet extends Toadlet {
@@ -29,7 +32,7 @@
private final BookmarkManager bookmarkManager;
private String cutedPath;
-
+
BookmarkEditorToadlet(HighLevelSimpleClient client, NodeClientCore core)
{
super(client);
@@ -37,39 +40,39 @@
this.bookmarkManager = core.bookmarkManager;
this.cutedPath = null;
}
-
+
private void addCategoryToList(BookmarkCategory cat, String path, HTMLNode list)
{
BookmarkItems items = cat.getItems();
-
- String edit = L10n.getString("BookmarkEditorToadlet.edit");
- String delete = L10n.getString("BookmarkEditorToadlet.delete");
- String cut = L10n.getString("BookmarkEditorToadlet.cut");
- String moveUp = L10n.getString("BookmarkEditorToadlet.moveUp");
- String moveDown = L10n.getString("BookmarkEditorToadlet.moveDown");
- String paste = L10n.getString("BookmarkEditorToadlet.paste");
- String addBookmark = L10n.getString("BookmarkEditorToadlet.addBookmark");
- String addCategory = L10n.getString("BookmarkEditorToadlet.addCategory");
-
+
+ final String edit = L10n.getString("BookmarkEditorToadlet.edit");
+ final String delete = L10n.getString("BookmarkEditorToadlet.delete");
+ final String cut = L10n.getString("BookmarkEditorToadlet.cut");
+ final String moveUp = L10n.getString("BookmarkEditorToadlet.moveUp");
+ final String moveDown = L10n.getString("BookmarkEditorToadlet.moveDown");
+ final String paste = L10n.getString("BookmarkEditorToadlet.paste");
+ final String addBookmark = L10n.getString("BookmarkEditorToadlet.addBookmark");
+ final String addCategory = L10n.getString("BookmarkEditorToadlet.addCategory");
+
for(int i = 0; i < items.size(); i++) {
- String itemPath = path + items.get(i).getName();
- HTMLNode li = new HTMLNode("li", "class","item" , items.get(i).getName());
+ String itemPath = URLEncoder.encode(path + items.get(i).getName());
+ HTMLNode li = new HTMLNode("li", "class", "item" , items.get(i).getName());
HTMLNode actions = new HTMLNode("span", "class", "actions");
actions.addChild("a", "href", "?action=edit&bookmark=" + itemPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/edit.png", edit, edit});
-
+
actions.addChild("a", "href", "?action=del&bookmark=" + itemPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/delete.png", delete, delete});
-
+
if(cutedPath == null)
actions.addChild("a", "href", "?action=cut&bookmark=" + itemPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/cut.png", cut, cut});
-
+
if(i != 0)
actions.addChild("a", "href", "?action=up&bookmark=" + itemPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/go-up.png", moveUp, moveUp});
-
+
if(i != items.size()-1)
actions.addChild("a", "href", "?action=down&bookmark=" + itemPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/go-down.png", moveDown, moveDown});
-
+
li.addChild(actions);
list.addChild(li);
}
@@ -77,32 +80,32 @@
BookmarkCategories cats = cat.getSubCategories();
for(int i = 0; i < cats.size(); i++) {
- String catPath = path + cats.get(i).getName() + "/";
-
+ String catPath = URLEncoder.encode(path + cats.get(i).getName() + "/");
+
HTMLNode subCat = list.addChild("li", "class", "cat", cats.get(i).getName());
HTMLNode actions = new HTMLNode("span", "class", "actions");
-
+
actions.addChild("a", "href", "?action=edit&bookmark=" + catPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/edit.png", edit, edit});
-
+
actions.addChild("a", "href", "?action=del&bookmark=" + catPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/delete.png", delete, delete});
-
+
actions.addChild("a", "href", "?action=addItem&bookmark=" + catPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/bookmark-new.png", addBookmark, addBookmark});
-
+
actions.addChild("a", "href", "?action=addCat&bookmark=" + catPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/folder-new.png", addCategory, addCategory});
-
+
if(cutedPath == null)
actions.addChild("a", "href", "?action=cut&bookmark=" + catPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/cut.png", cut, cut});
-
+
if(i != 0)
actions.addChild("a", "href", "?action=up&bookmark=" + catPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/go-up.png", moveUp, moveUp});
-
+
if(i != cats.size() -1)
actions.addChild("a", "href", "?action=down&bookmark=" + catPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/go-down.png", moveDown, moveDown});
if(cutedPath != null && ! catPath.startsWith(cutedPath) && ! catPath.equals(bookmarkManager.parentPath(cutedPath)))
actions.addChild("a", "href", "?action=paste&bookmark=" + catPath).addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/paste.png", paste, paste});
-
+
subCat.addChild(actions);
if(cats.get(i).size() != 0)
addCategoryToList(cats.get(i), catPath, list.addChild("li").addChild("ul"));
@@ -112,111 +115,120 @@
public HTMLNode getBookmarksList()
{
HTMLNode bookmarks = new HTMLNode("ul", "id", "bookmarks");
-
- HTMLNode root = bookmarks.addChild("li", "class", "cat,root", "/");
+
+ HTMLNode root = bookmarks.addChild("li", "class", "cat root", "/");
HTMLNode actions = new HTMLNode("span", "class", "actions");
String addBookmark = L10n.getString("BookmarkEditorToadlet.addBookmark");
String addCategory = L10n.getString("BookmarkEditorToadlet.addCategory");
String paste = L10n.getString("BookmarkEditorToadlet.paste");
actions.addChild("a", "href", "?action=addItem&bookmark=/").addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/bookmark-new.png", addBookmark, addBookmark});
actions.addChild("a", "href", "?action=addCat&bookmark=/").addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/folder-new.png", addCategory, addCategory});
-
+
if(cutedPath != null && ! "/".equals(bookmarkManager.parentPath(cutedPath)))
actions.addChild("a", "href", "?action=paste&bookmark=/").addChild("img", new String[] {"src", "alt", "title"}, new String[] {"/static/icon/paste.png", paste, paste});
-
+
root.addChild(actions);
- addCategoryToList(bookmarkManager.getMainCategory(), "/", root.addChild("li").addChild("ul"));
-
+ addCategoryToList(bookmarkManager.getMainCategory(), "/", root.addChild("ul"));
+
return bookmarks;
}
-
+
public void handleGet(URI uri, HTTPRequest req, ToadletContext ctx)
- throws ToadletContextClosedException, IOException
+ throws ToadletContextClosedException, IOException
{
-
+
String editorTitle = L10n.getString("BookmarkEditorToadlet.title");
String error = L10n.getString("BookmarkEditorToadlet.error");
HTMLNode pageNode = ctx.getPageMaker().getPageNode(editorTitle, ctx);
HTMLNode content = ctx.getPageMaker().getContentNode(pageNode);
-
+
if (req.getParam("action").length() > 0 && req.getParam("bookmark").length() > 0) {
String action = req.getParam("action");
- String bookmarkPath = req.getParam("bookmark");
+ String bookmarkPath;
+ try {
+ bookmarkPath = URLDecoder.decode(req.getParam("bookmark"), false);
+ } catch (URLEncodedFormatException e) {
+ HTMLNode errorBox = content.addChild(ctx.getPageMaker().getInfobox("infobox-error", error));
+ errorBox.addChild("#", L10n.getString("BookmarkEditorToadlet.urlDecodeError"));
+ writeHTMLReply(ctx, 200, "OK", pageNode.generate());
+ return;
+ }
Bookmark bookmark;
-
+
if (bookmarkPath.endsWith("/"))
bookmark = bookmarkManager.getCategoryByPath(bookmarkPath);
else
bookmark = bookmarkManager.getItemByPath(bookmarkPath);
-
+
if(bookmark == null) {
HTMLNode errorBox = content.addChild(ctx.getPageMaker().getInfobox("infobox-error"