From david at sowder.com Tue Jan 1 15:23:49 2008 From: david at sowder.com (David Sowder) Date: Tue, 01 Jan 2008 09:23:49 -0600 Subject: [freenet-dev] [freenet-cvs] r16825 - trunk/freenet/src/freenet/node In-Reply-To: <20071228165029.8866D47AEA3@freenetproject.org> References: <20071228165029.8866D47AEA3@freenetproject.org> Message-ID: <477A5B05.6010903@sowder.com> Do we want to print the null? robert at emu.freenetproject.org wrote: > Author: robert > Date: 2007-12-28 16:50:28 +0000 (Fri, 28 Dec 2007) > New Revision: 16825 > > Modified: > trunk/freenet/src/freenet/node/SSKInsertSender.java > Log: > logging > > > Modified: trunk/freenet/src/freenet/node/SSKInsertSender.java > =================================================================== > --- trunk/freenet/src/freenet/node/SSKInsertSender.java 2007-12-27 21:56:35 UTC (rev 16824) > +++ trunk/freenet/src/freenet/node/SSKInsertSender.java 2007-12-28 16:50:28 UTC (rev 16825) > @@ -309,7 +309,7 @@ > if (msg == null) { > // Timeout :( > // Fairly serious problem > - Logger.error(this, "Timeout (" + next + ") after Accepted in insert"); > + Logger.error(this, "Timeout (" + msg + ") after Accepted in insert; to ("+next+")"); > // Terminal overload > // Try to propagate back to source > next.localRejectedOverload("AfterInsertAcceptedTimeout"); > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > -- David R. Sowder Linux Systems Administrator (Software Systems Specialist II) Office of Information Technology University of Texas at Arlington Work: 817-272-1081 davids at uta.edu http://www.uta.edu/ Personal: david at sowder.com http://david.sowder.com/ From toad at amphibian.dyndns.org Wed Jan 2 12:57:18 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 2 Jan 2008 12:57:18 +0000 Subject: [freenet-dev] [freenet-cvs] r16825 - trunk/freenet/src/freenet/node In-Reply-To: <477A5B05.6010903@sowder.com> References: <20071228165029.8866D47AEA3@freenetproject.org> <477A5B05.6010903@sowder.com> Message-ID: <200801021257.18435.toad@amphibian.dyndns.org> On Tuesday 01 January 2008 15:23, David Sowder wrote: > Do we want to print the null? Probably a good idea. > > robert at emu.freenetproject.org wrote: > > Author: robert > > Date: 2007-12-28 16:50:28 +0000 (Fri, 28 Dec 2007) > > New Revision: 16825 > > > > Modified: > > trunk/freenet/src/freenet/node/SSKInsertSender.java > > Log: > > logging > > > > > > Modified: trunk/freenet/src/freenet/node/SSKInsertSender.java > > =================================================================== > > --- trunk/freenet/src/freenet/node/SSKInsertSender.java 2007-12-27 21:56:35 UTC (rev 16824) > > +++ trunk/freenet/src/freenet/node/SSKInsertSender.java 2007-12-28 16:50:28 UTC (rev 16825) > > @@ -309,7 +309,7 @@ > > if (msg == null) { > > // Timeout :( > > // Fairly serious problem > > - Logger.error(this, "Timeout (" + next + ") after Accepted in insert"); > > + Logger.error(this, "Timeout (" + msg + ") after Accepted in insert; to ("+next+")"); > > // Terminal overload > > // Try to propagate back to source > > next.localRejectedOverload("AfterInsertAcceptedTimeout"); > > > > _______________________________________________ > > cvs mailing list > > cvs at freenetproject.org > > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > > > > -- > David R. Sowder > Linux Systems Administrator (Software Systems Specialist II) > Office of Information Technology > University of Texas at Arlington > Work: 817-272-1081 davids at uta.edu http://www.uta.edu/ > Personal: david at sowder.com http://david.sowder.com/ > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/3e810fe1/attachment.pgp From toad at amphibian.dyndns.org Wed Jan 2 12:58:45 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 2 Jan 2008 12:58:45 +0000 Subject: [freenet-dev] [freenet-cvs] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <20071229113933.GD4199@freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229110517.GB4199@freenetproject.org> <20071229113933.GD4199@freenetproject.org> Message-ID: <200801021258.46217.toad@amphibian.dyndns.org> On Saturday 29 December 2007 11:39, Florent Daigni?re wrote: > * Florent Daigni?re [2007-12-29 12:05:17]: > > > * robert at freenetproject.org [2007-12-29 01:46:31]: > > > > > Author: robert > > > Date: 2007-12-29 01:46:31 +0000 (Sat, 29 Dec 2007) > > > New Revision: 16834 > > > > > > Modified: > > > trunk/freenet/src/freenet/node/PeerNode.java > > > Log: > > > maybe help wont-fetch-ark deadlock #2 > > > > > > > > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > > > =================================================================== > > > --- trunk/freenet/src/freenet/node/PeerNode.java 2007-12-29 01:41:06 UTC (rev 16833) > > > +++ trunk/freenet/src/freenet/node/PeerNode.java 2007-12-29 01:46:31 UTC (rev 16834) > > > @@ -2782,7 +2782,7 @@ > > > > > > synchronized void updateShouldDisconnectNow() { > > > //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH THE NODE > > > - if (isConnected()) { > > > + if (isConnected() || verifiedIncompatibleOlderVersion || verifiedIncompatibleNewerVersion) { > > > verifiedIncompatibleOlderVersion = forwardInvalidVersion(); > > > verifiedIncompatibleNewerVersion = reverseInvalidVersion(); > > > } > > > > I suggest you call isRoutable() instead > > > > NextGen$ > > Hmm, on a latter thought, I don't understand why it helps... nor why we > are calling it from PacketSender > > What about getting rid of both the "if" branch and the call in > PacketSender ? Why not just use if(isConnected()) { ... } ? As far as I can see that is correct: if we're not connected, we don't care about verified*. > > NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/60d9ba92/attachment.pgp From nextgens at freenetproject.org Wed Jan 2 13:38:04 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Wed, 2 Jan 2008 14:38:04 +0100 Subject: [freenet-dev] [freenet-cvs] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <200801021258.46217.toad@amphibian.dyndns.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229110517.GB4199@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801021258.46217.toad@amphibian.dyndns.org> Message-ID: <20080102133803.GA4775@freenetproject.org> * Matthew Toseland [2008-01-02 12:58:45]: > On Saturday 29 December 2007 11:39, Florent Daigni?re wrote: > > * Florent Daigni?re [2007-12-29 12:05:17]: > > > > > * robert at freenetproject.org [2007-12-29 > 01:46:31]: > > > > > > > Author: robert > > > > Date: 2007-12-29 01:46:31 +0000 (Sat, 29 Dec 2007) > > > > New Revision: 16834 > > > > > > > > Modified: > > > > trunk/freenet/src/freenet/node/PeerNode.java > > > > Log: > > > > maybe help wont-fetch-ark deadlock #2 > > > > > > > > > > > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > > > > =================================================================== > > > > --- trunk/freenet/src/freenet/node/PeerNode.java 2007-12-29 01:41:06 UTC > (rev 16833) > > > > +++ trunk/freenet/src/freenet/node/PeerNode.java 2007-12-29 01:46:31 UTC > (rev 16834) > > > > @@ -2782,7 +2782,7 @@ > > > > > > > > synchronized void updateShouldDisconnectNow() { > > > > //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH THE > NODE > > > > - if (isConnected()) { > > > > + if (isConnected() || verifiedIncompatibleOlderVersion || > verifiedIncompatibleNewerVersion) { > > > > verifiedIncompatibleOlderVersion = forwardInvalidVersion(); > > > > verifiedIncompatibleNewerVersion = reverseInvalidVersion(); > > > > } > > > > > > I suggest you call isRoutable() instead > > > > > > NextGen$ > > > > Hmm, on a latter thought, I don't understand why it helps... nor why we > > are calling it from PacketSender > > > > What about getting rid of both the "if" branch and the call in > > PacketSender ? > > Why not just use if(isConnected()) { ... } ? As far as I can see that is > correct: if we're not connected, we don't care about verified*. You're suggesting to revert robert's patch, wich is fine by me... as I said, I don't understand why it helps NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/2e8eff83/attachment.pgp From toad at amphibian.dyndns.org Wed Jan 2 12:56:11 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 2 Jan 2008 12:56:11 +0000 Subject: [freenet-dev] [freenet-cvs] r16775 - in trunk/freenet/src/freenet: clients/http crypt io l10n node node/fcp In-Reply-To: <20071225011902.GA5185@freenetproject.org> References: <20071222001228.E0436479FBC@freenetproject.org> <20071225011902.GA5185@freenetproject.org> Message-ID: <200801021256.11404.toad@amphibian.dyndns.org> Ian: example of why we need DVCS IMHO. I committed a patch from ET on Frost, because I want anonymous contributors, and to encourage a new dev. I did a basic code review (and gave input on architecture, this is the second patch to be committed), now nextgens is quibbling about it... On Tuesday 25 December 2007 01:19, Florent Daigni?re wrote: > * toad at freenetproject.org [2007-12-22 00:12:28]: > > > Author: toad > > Date: 2007-12-22 00:12:28 +0000 (Sat, 22 Dec 2007) > > New Revision: 16775 > > > > Added: > > trunk/freenet/src/freenet/crypt/SSL.java > > trunk/freenet/src/freenet/io/SSLNetworkInterface.java > > Modified: > > trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java > > trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties > > trunk/freenet/src/freenet/node/NodeStarter.java > > trunk/freenet/src/freenet/node/TextModeClientInterfaceServer.java > > trunk/freenet/src/freenet/node/fcp/FCPServer.java > > Log: > > Patch from ET at mj+bSV4hxRMtCj9fcwy4Ww9_3mc on Frost: Add SSL support for FCP, HTTP, TMCI. > > Request testing! > > > > Why has it been commited and released ? > 1) it depends on sun.security.x509.X500Name Which is bad because...? > 2) errors sent back by the config. framework aren't > internationalized AFAIK this is a problem in many places in Fred. > 3) they are obvious typos and french comments! Okay, French comments are bad. I did read it before I committed it, it seemed okay, but maybe I was asleep at that point. > > Don't get me wrong: contributions are welcome... but they should meet a > given quality standard to get merged and deployed. How do you propose we discuss such contributions? Did you see it on Frost? Our current protocol is to throw everything into trunk, 99% of the time. Maybe it should have gone into a branch, but IMHO the ideal is a DVCS so I can just post the (freenet-based) url on devl. I don't suppose you'd be interested in getting that working? > > NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/b3dca549/attachment.pgp From toad at amphibian.dyndns.org Wed Jan 2 12:56:11 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 2 Jan 2008 12:56:11 +0000 Subject: [freenet-dev] [freenet-cvs] r16775 - in trunk/freenet/src/freenet: clients/http crypt io l10n node node/fcp In-Reply-To: <20071225011902.GA5185@freenetproject.org> References: <20071222001228.E0436479FBC@freenetproject.org> <20071225011902.GA5185@freenetproject.org> Message-ID: <200801021256.11404.toad@amphibian.dyndns.org> Ian: example of why we need DVCS IMHO. I committed a patch from ET on Frost, because I want anonymous contributors, and to encourage a new dev. I did a basic code review (and gave input on architecture, this is the second patch to be committed), now nextgens is quibbling about it... On Tuesday 25 December 2007 01:19, Florent Daigni?re wrote: > * toad at freenetproject.org [2007-12-22 00:12:28]: > > > Author: toad > > Date: 2007-12-22 00:12:28 +0000 (Sat, 22 Dec 2007) > > New Revision: 16775 > > > > Added: > > trunk/freenet/src/freenet/crypt/SSL.java > > trunk/freenet/src/freenet/io/SSLNetworkInterface.java > > Modified: > > trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java > > trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties > > trunk/freenet/src/freenet/node/NodeStarter.java > > trunk/freenet/src/freenet/node/TextModeClientInterfaceServer.java > > trunk/freenet/src/freenet/node/fcp/FCPServer.java > > Log: > > Patch from ET at mj+bSV4hxRMtCj9fcwy4Ww9_3mc on Frost: Add SSL support for FCP, HTTP, TMCI. > > Request testing! > > > > Why has it been commited and released ? > 1) it depends on sun.security.x509.X500Name Which is bad because...? > 2) errors sent back by the config. framework aren't > internationalized AFAIK this is a problem in many places in Fred. > 3) they are obvious typos and french comments! Okay, French comments are bad. I did read it before I committed it, it seemed okay, but maybe I was asleep at that point. > > Don't get me wrong: contributions are welcome... but they should meet a > given quality standard to get merged and deployed. How do you propose we discuss such contributions? Did you see it on Frost? Our current protocol is to throw everything into trunk, 99% of the time. Maybe it should have gone into a branch, but IMHO the ideal is a DVCS so I can just post the (freenet-based) url on devl. I don't suppose you'd be interested in getting that working? > > NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/b3dca549/attachment-0001.pgp From nextgens at freenetproject.org Wed Jan 2 15:56:08 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Wed, 2 Jan 2008 16:56:08 +0100 Subject: [freenet-dev] [freenet-cvs] r16775 - in trunk/freenet/src/freenet: clients/http crypt io l10n node node/fcp In-Reply-To: <200801021256.11404.toad@amphibian.dyndns.org> References: <20071222001228.E0436479FBC@freenetproject.org> <20071225011902.GA5185@freenetproject.org> <200801021256.11404.toad@amphibian.dyndns.org> Message-ID: <20080102155608.GC4775@freenetproject.org> * Matthew Toseland [2008-01-02 12:56:11]: > On Tuesday 25 December 2007 01:19, Florent Daigni?re wrote: > > * toad at freenetproject.org [2007-12-22 00:12:28]: > > > > > Author: toad > > > Date: 2007-12-22 00:12:28 +0000 (Sat, 22 Dec 2007) > > > New Revision: 16775 > > > > > > Added: > > > trunk/freenet/src/freenet/crypt/SSL.java > > > trunk/freenet/src/freenet/io/SSLNetworkInterface.java > > > Modified: > > > trunk/freenet/src/freenet/clients/http/SimpleToadletServer.java > > > trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties > > > trunk/freenet/src/freenet/node/NodeStarter.java > > > trunk/freenet/src/freenet/node/TextModeClientInterfaceServer.java > > > trunk/freenet/src/freenet/node/fcp/FCPServer.java > > > Log: > > > Patch from ET at mj+bSV4hxRMtCj9fcwy4Ww9_3mc on Frost: Add SSL support for > FCP, HTTP, TMCI. > > > Request testing! > > > > > > > Why has it been commited and released ? > > 1) it depends on sun.security.x509.X500Name > > Which is bad because...? It breaks freejvm support (freenet is reported to be working on recent gij) > > > 2) errors sent back by the config. framework aren't > > internationalized > > AFAIK this is a problem in many places in Fred. True but introducing more is definitely not going to help ;) > > > 3) they are obvious typos and french comments! > > Okay, French comments are bad. I did read it before I committed it, it seemed > okay, but maybe I was asleep at that point. > > > > Don't get me wrong: contributions are welcome... but they should meet a > > given quality standard to get merged and deployed. > > How do you propose we discuss such contributions? Well, it's clear that we aren't ready to deal with "big" anonymous contributions... but so far it hasn't been a worry because we don't have many anon. contributors... and most of the time they publish trivial patches. >Did you see it on Frost? Yes, and there too I was the first one to criticize it > Our current protocol is to throw everything into trunk, 99% of the time. Maybe it > should have gone into a branch, I think it would have been better for it to remain in a branch, yes. I have no problem with such patches being in our trunk... but you shouldn't release right after they have been introduced. > but IMHO the ideal is a DVCS so I can just > post the (freenet-based) url on devl. I don't suppose you'd be interested in > getting that working? I have no time to do it atm... but will have a look after I've passed my exams. I'm reluctant to use a DSCM in our case for a few reasons: 1) they are more difficult to use: that will be one additional fence each of our contributors will have to pass before he can start to contribute 2) they are "young": more likely to be prone to bugs and stuffs like that (keep in mind that we did experience data corruption with svn... which is believed to be robust!) 3) last time I checked they weren't any "good" GUIs: you are the one who told me "as long as there is an eclipse integration I don't mind what we are using" back when I made you switch from CVS to SVN ;) If you're still convinced that it's the way to go, I'll have a closer look to what's doable... We did talk about it at the gSoC mentor summit and because of my point (1) we ended up concluding "maybe latter". NextGen$ PS: a point I forgot to state in my previous email : from an architectural point of view, I don't think that we should have "one" fproxy but more... One with and one without ssl support, possibly running on different port numbers. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/fe3274d2/attachment.pgp From robert at emu.freenetproject.org Wed Jan 2 17:22:17 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Wed, 2 Jan 2008 11:22:17 -0600 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <20071229113933.GD4199@freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229110517.GB4199@freenetproject.org> <20071229113933.GD4199@freenetproject.org> Message-ID: On Dec 29, 2007, at 5:39 AM, Florent Daigni?re wrote: >>> synchronized void updateShouldDisconnectNow() { >>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH >>> THE NODE >>> - if (isConnected()) { >>> + if (isConnected() || verifiedIncompatibleOlderVersion || >>> verifiedIncompatibleNewerVersion) { >>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); >>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); >>> } >> >> I suggest you call isRoutable() instead >> >> NextGen$ > > Hmm, on a latter thought, I don't understand why it helps... nor why > we > are calling it from PacketSender > > What about getting rid of both the "if" branch and the call in > PacketSender ? In PacketSender this function is called ~precisely~ at the version transition time so that all peers will be re-evaluated concerning compatible versioning; this seems like an odd place to stick this code, but also updateShouldDisconnectNow() does not explicitly disconnect the peer node, it only updates the 'incompatible' flags. Concerning the if branch I added, the intended behaviour change is that this function should only (1) allow the disconnection of a connected peer by setting the incompatible flag, or (2) remove an incompatible flag (e.g. on the fetch of an ark). Now, if the nodes can handshake, this code has probably has little effect, as it simply further restricts one of only two places that the incompatible flags are set (the other being on a handshake). What it will do is if the nodes can't talk to each other (like in my re-keyed node case), is not show 'TOO_OLD'/'TOO_NEW', but instead 'DISCONNECTED'. And I suppose in the same error case... if the node is restarted, it would never be able to verify that the peer is too old/ new and fetch the ark (keeping the version number up-to-date). Really this change just brings the functionality of the incompatibility flags closer to the comments thereon: that they are only set if we have verified incompatibility through a handshake. In practice, it appears that they could still be set upon receiving a new reference (ark?) as well. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080102/0836e813/attachment.htm From robert at emu.freenetproject.org Wed Jan 2 17:30:18 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Wed, 2 Jan 2008 11:30:18 -0600 Subject: [freenet-dev] [freenet-cvs] r16825 - trunk/freenet/src/freenet/node In-Reply-To: <200801021257.18435.toad@amphibian.dyndns.org> References: <20071228165029.8866D47AEA3@freenetproject.org> <477A5B05.6010903@sowder.com> <200801021257.18435.toad@amphibian.dyndns.org> Message-ID: On Jan 2, 2008, at 6:57 AM, Matthew Toseland wrote: > On Tuesday 01 January 2008 15:23, David Sowder wrote: >> Do we want to print the null? > > Probably a good idea. It does seem kind of weird perhaps (as msg will always be null), but it makes the logging match that in CHKInsertSender, and when reading it in the log files it makes more sense (than received a... ? peernode?!). The CHK log statement can also have a rejected/overload- timeout or something. -- Robert Hailey >> if (msg == null) { >> // Timeout :( >> // Fairly serious problem >> - Logger.error(this, "Timeout (" + next + ") after Accepted in >> insert"); >> + Logger.error(this, "Timeout (" + msg + ") after Accepted in >> insert; to ("+next+")"); -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080102/2a299ce7/attachment.htm From robert at emu.freenetproject.org Wed Jan 2 18:15:16 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Wed, 2 Jan 2008 12:15:16 -0600 Subject: [freenet-dev] [freenet-cvs] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <20080102133803.GA4775@freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229110517.GB4199@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801021258.46217.toad@amphibian.dyndns.org> <20080102133803.GA4775@freenetproject.org> Message-ID: <199BD599-D8DF-40FB-975B-2BF94854BF16@emu.freenetproject.org> On Jan 2, 2008, at 7:38 AM, Florent Daigni?re wrote: >>>>> synchronized void updateShouldDisconnectNow() { >>>>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH >>>>> THE NODE >>>>> - if (isConnected()) { >>>>> + if (isConnected() || verifiedIncompatibleOlderVersion || >>>>> verifiedIncompatibleNewerVersion) { >>>>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); >>>>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); >>>>> } >>>> >>>> I suggest you call isRoutable() instead >>>> >>>> NextGen$ >>> >>> Hmm, on a latter thought, I don't understand why it helps... nor >>> why we >>> are calling it from PacketSender >>> >>> What about getting rid of both the "if" branch and the call in >>> PacketSender ? >> >> Why not just use if(isConnected()) { ... } ? As far as I can see >> that is >> correct: if we're not connected, we don't care about verified*. > > You're suggesting to revert robert's patch, wich is fine by me... as I > said, I don't understand why it helps > > NextGen$ This function is called at various times (connected or not). If the only condition baring updating the 'verified' flags is: "isConnected()"... then the added condition could SET the verified- incompatible flags but could not UNSET them unless connected. The purpose for adding the extra OR's to the conditional is to allow this function to clear the verified-incompatible flags even if not connected (e.g. new ark fetched, node has a new version, clear the verified-flag). I have had what appears to be this 'deadlock' twice, the first with a laptop (whose IP and node-version changed at the same time), and it is this: (1) last-seen node version is incompatible (but before r16834, would set 'verified') (2) attempt handshakes, but don't fetch ark (3) IP has changed, so can't handshake (4) wont get new IP (ark), because older version A better solution to this might simply be to fetch arks even on version incompatibility, but (for caution) behavior #2 seemed intentional (dont fetch ark on incompatible version), whereas #1 did not, and brought it closer to the comments (i.e. don't set 'verified' unless handshake). -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080102/0806e301/attachment.htm From freenet-devl at david.sowder.com Wed Jan 2 18:30:59 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Wed, 02 Jan 2008 12:30:59 -0600 Subject: [freenet-dev] [freenet-cvs] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <199BD599-D8DF-40FB-975B-2BF94854BF16@emu.freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229110517.GB4199@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801021258.46217.toad@amphibian.dyndns.org> <20080102133803.GA4775@freenetproject.org> <199BD599-D8DF-40FB-975B-2BF94854BF16@emu.freenetproject.org> Message-ID: <477BD863.50803@david.sowder.com> I'm not sure of the thinking of others regarding changes to the verifiedIncompatible variables since, but my thinking in my initial implementation was that they would only be set after a handshake, thus we already had a connected to them and ARKs weren't needed. That won't change until the peer re-handshakes after restarting with a different version of the node's software. Robert Hailey wrote: > > On Jan 2, 2008, at 7:38 AM, Florent Daigni?re wrote: > >>>>>> synchronized void updateShouldDisconnectNow() { >>>>>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH >>>>>> THE NODE >>>>>> - if (isConnected()) { >>>>>> + if (isConnected() || verifiedIncompatibleOlderVersion >>>>>> || verifiedIncompatibleNewerVersion) { >>>>>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); >>>>>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); >>>>>> } >>>>> >>>>> I suggest you call isRoutable() instead >>>>> >>>>> NextGen$ >>>> >>>> Hmm, on a latter thought, I don't understand why it helps... nor why we >>>> are calling it from PacketSender >>>> >>>> What about getting rid of both the "if" branch and the call in >>>> PacketSender ? >>> >>> Why not just use if(isConnected()) { ... } ? As far as I can see >>> that is >>> correct: if we're not connected, we don't care about verified*. >> >> You're suggesting to revert robert's patch, wich is fine by me... as I >> said, I don't understand why it helps >> >> NextGen$ > > This function is called at various times (connected or not). If the > only condition baring updating the 'verified' flags is: > "isConnected()"... then the added condition could SET the > verified-incompatible flags but could not UNSET them unless connected. > The purpose for adding the extra OR's to the conditional is to allow > this function to clear the verified-incompatible flags even if not > connected (e.g. new ark fetched, node has a new version, clear the > verified-flag). > > I have had what appears to be this 'deadlock' twice, the first with a > laptop (whose IP and node-version changed at the same time), and it is > this: > (1) last-seen node version is incompatible (but before r16834, would > set 'verified') > (2) attempt handshakes, but don't fetch ark > (3) IP has changed, so can't handshake > (4) wont get new IP (ark), because older version > > A better solution to this might simply be to fetch arks even on > version incompatibility, but (for caution) behavior #2 seemed > intentional (dont fetch ark on incompatible version), whereas #1 did > not, and brought it closer to the comments (i.e. don't set 'verified' > unless handshake). > > -- > Robert Hailey > > ------------------------------------------------------------------------ > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From toad at amphibian.dyndns.org Wed Jan 2 21:37:46 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 2 Jan 2008 21:37:46 +0000 Subject: [freenet-dev] =?utf-8?q?=5Bfreenet-cvs=5D_r16834_-=09trunk/freene?= =?utf-8?q?t/src/freenet/node?= In-Reply-To: <199BD599-D8DF-40FB-975B-2BF94854BF16@emu.freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20080102133803.GA4775@freenetproject.org> <199BD599-D8DF-40FB-975B-2BF94854BF16@emu.freenetproject.org> Message-ID: <200801022137.46575.toad@amphibian.dyndns.org> On Wednesday 02 January 2008 18:15, Robert Hailey wrote: > > On Jan 2, 2008, at 7:38 AM, Florent Daigni?re wrote: > > >>>>> synchronized void updateShouldDisconnectNow() { > >>>>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH > >>>>> THE NODE > >>>>> - if (isConnected()) { > >>>>> + if (isConnected() || verifiedIncompatibleOlderVersion || > >>>>> verifiedIncompatibleNewerVersion) { > >>>>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); > >>>>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); > >>>>> } > >>>> > >>>> I suggest you call isRoutable() instead > >>>> > >>>> NextGen$ > >>> > >>> Hmm, on a latter thought, I don't understand why it helps... nor > >>> why we > >>> are calling it from PacketSender > >>> > >>> What about getting rid of both the "if" branch and the call in > >>> PacketSender ? > >> > >> Why not just use if(isConnected()) { ... } ? As far as I can see > >> that is > >> correct: if we're not connected, we don't care about verified*. > > > > You're suggesting to revert robert's patch, wich is fine by me... as I > > said, I don't understand why it helps > > > > NextGen$ > > This function is called at various times (connected or not). If the > only condition baring updating the 'verified' flags is: > "isConnected()"... then the added condition could SET the verified- > incompatible flags but could not UNSET them unless connected. The > purpose for adding the extra OR's to the conditional is to allow this > function to clear the verified-incompatible flags even if not > connected (e.g. new ark fetched, node has a new version, clear the > verified-flag). Okay. Why is this a problem? > > I have had what appears to be this 'deadlock' twice, the first with a > laptop (whose IP and node-version changed at the same time), and it is > this: > (1) last-seen node version is incompatible (but before r16834, would > set 'verified') > (2) attempt handshakes, but don't fetch ark Why does it not fetch the ARK? It should fetch the ARK even if it is incompatible. > (3) IP has changed, so can't handshake > (4) wont get new IP (ark), because older version This is the bug afaics. > > A better solution to this might simply be to fetch arks even on > version incompatibility, but (for caution) behavior #2 seemed > intentional (dont fetch ark on incompatible version), whereas #1 did > not, and brought it closer to the comments (i.e. don't set 'verified' > unless handshake). I really don't see any reason not to fetch the ARK just because it was incompatible last time. That's a bug, it's a connectivity problem. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/1a8982fc/attachment.pgp From toad at amphibian.dyndns.org Wed Jan 2 21:40:34 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 2 Jan 2008 21:40:34 +0000 Subject: [freenet-dev] =?iso-8859-1?q?=5Bfreenet-cvs=5D_r16834=09-=09trunk?= =?iso-8859-1?q?/freenet/src/freenet/node?= In-Reply-To: <477BD863.50803@david.sowder.com> References: <20071229014631.C5D7847A6AE@freenetproject.org> <199BD599-D8DF-40FB-975B-2BF94854BF16@emu.freenetproject.org> <477BD863.50803@david.sowder.com> Message-ID: <200801022140.35033.toad@amphibian.dyndns.org> On Wednesday 02 January 2008 18:30, David Sowder wrote: > I'm not sure of the thinking of others regarding changes to the > verifiedIncompatible variables since, but my thinking in my initial > implementation was that they would only be set after a handshake, thus > we already had a connected to them and ARKs weren't needed. That won't > change until the peer re-handshakes after restarting with a different > version of the node's software. I don't follow - clearly if isConnected(), we don't need the ARK. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/e11f0e01/attachment.pgp From toad at amphibian.dyndns.org Wed Jan 2 21:44:58 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 2 Jan 2008 21:44:58 +0000 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229113933.GD4199@freenetproject.org> Message-ID: <200801022144.58447.toad@amphibian.dyndns.org> On Wednesday 02 January 2008 17:22, Robert Hailey wrote: > > On Dec 29, 2007, at 5:39 AM, Florent Daigni?re wrote: > > >>> synchronized void updateShouldDisconnectNow() { > >>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH > >>> THE NODE > >>> - if (isConnected()) { > >>> + if (isConnected() || verifiedIncompatibleOlderVersion || > >>> verifiedIncompatibleNewerVersion) { > >>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); > >>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); > >>> } > >> > >> I suggest you call isRoutable() instead > >> > >> NextGen$ > > > > Hmm, on a latter thought, I don't understand why it helps... nor why > > we > > are calling it from PacketSender > > > > What about getting rid of both the "if" branch and the call in > > PacketSender ? > > In PacketSender this function is called ~precisely~ at the version > transition time so that all peers will be re-evaluated concerning > compatible versioning; this seems like an odd place to stick this > code, but also updateShouldDisconnectNow() does not explicitly > disconnect the peer node, it only updates the 'incompatible' flags. We don't want to disconnect. We want to stay connected and to not route requests to the node or accept them from it. We will still accept e.g. Update Over Mandatory traffic. > > Concerning the if branch I added, the intended behaviour change is > that this function should only (1) allow the disconnection of a > connected peer by setting the incompatible flag, No, we don't want to disconnect. We just want to make it non-routable, by updating verified*. > or (2) remove an > incompatible flag (e.g. on the fetch of an ark). > > Now, if the nodes can handshake, this code has probably has little > effect, as it simply further restricts one of only two places that the > incompatible flags are set (the other being on a handshake). What it > will do is if the nodes can't talk to each other (like in my re-keyed > node case), is not show 'TOO_OLD'/'TOO_NEW', but instead > 'DISCONNECTED'. So you are setting verified* to false, because it's not verified? > And I suppose in the same error case... if the node is > restarted, it would never be able to verify that the peer is too old/ > new and fetch the ark (keeping the version number up-to-date). Again, we should always fetch the ARK. > > Really this change just brings the functionality of the > incompatibility flags closer to the comments thereon: that they are > only set if we have verified incompatibility through a handshake. In > practice, it appears that they could still be set upon receiving a new > reference (ark?) as well. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/e532040c/attachment.pgp From toad at amphibian.dyndns.org Wed Jan 2 23:52:45 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 2 Jan 2008 23:52:45 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <47779AEF.6080305@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200712210025.29457.toad@amphibian.dyndns.org> <47779AEF.6080305@cs.ucl.ac.uk> Message-ID: <200801022352.54143.toad@amphibian.dyndns.org> On Sunday 30 December 2007 13:19, Michael Rogers wrote: > > The idea was to pick two random nodes from within the cell, and route through > > them - choosing a new tunnel for every request. > > Why choose a new tunnel for every request? If an attacker inside the > cell can somehow link messages leaving the cell with messages passing > through his node, he can gather one predecessor attack sample per > intercepted message. True, but: - It is fast. - It is simple. - It should be possible with such a simple request/response to make it extremely difficult for two nodes to realise they are on the same path, except in the case where they are actually adjacent. If: - The two nodes on the path controlled by the attacker are not adjacent, - We have a separate return path included in the routing information, - Each hop waits to receive the full reply before forwarding it, Then it will be very difficult for an attacker to identify that the two nodes are connected afaics: with bidirectional tunnels, his only leverage is the reply time, but with unidirectional tunnels he loses even that. If the two nodes *are* adjacent, we can severely limit the information available: The last node is the second chosen node (assuming we choose two nodes including the exit node). The second-last node is either a node between the first chosen node and the second chosen node, or is in fact the first chosen node, in which case its predecessor is a back-bearing towards the originator. However, we can prevent this by ensuring there is always at least one hop between the first chosen node and the second chosen node. Since this does not give any information on the originator, it should be a safe addition. The separate return path obviously will be unreliable - any node along it could refuse it. This is the same for the outward path: any node could refuse the path, or sabotage it, and it will be impossible to establish which node is at fault since any node can always blame a later node. However, it *is* known that it is one of the nodes along that path that is the problem, and this may be enough. With enough redundancy it might even be possible to make passive requests work through this, although the main concern here is the real time phase of a request. Perhaps I am missing something critical about onion routing here, but what? > > > The one big weakness is that cells would have to be relatively small > > to keep global data available. The complication (IMHO solvable) is bogus > > topology attacks. > > And bogus public key attacks (the nodes beyond my neighbour might really > exist, but my neighbour might MITM their public keys in order to link > traffic passing through his node with traffic exiting the cell). When > you say solvable, what do you have in mind? Each node publishes its links, including its pubkeys, their pubkey hashes, and a signature. This is relayed to some distance. We compute a credibility value for each node on the graph by starting with a value of 1.0 on each node we are a direct peer of, and dividing it by the number of peers on each node going outwards, adding up the values from different branches. We can then ignore nodes which have a low credibility value. Now, we may not be able to do exactly this, if we have to have semi-permanent cell structures, but it should be possible to find an algorithm to serve this purpose (eigenvector centrality? TrustRank etc?). > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080102/7950bb26/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 00:00:05 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 00:00:05 +0000 Subject: [freenet-dev] Why getting rid of HTL is bad was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <47779AEF.6080305@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200712210025.29457.toad@amphibian.dyndns.org> <47779AEF.6080305@cs.ucl.ac.uk> Message-ID: <200801030000.13355.toad@amphibian.dyndns.org> On Sunday 30 December 2007 13:19, Michael Rogers wrote: > Matthew Toseland wrote: > > - Not compatible afaics with ULPRs / failure tables. ULPRs stop requests if > > we've routed them already (for some period), with the same HTL or less. > > Good point, and I guess the same would be true of request coalescing. > Maybe we could get a similar effect by using the weighted coin to decide > whether to coalesce or forward the new request. > > > - Load management issues maybe. > > Do we currently use the hop counter for any load management decisions? No, but it would make the request time variance much higher, especially for inserts and unsuccessful requests (which can potentially visit the entire network). > > > - It's a good "invisible" DoS on a single request. > > Sorry, I don't see what you mean - the hop counter can be tampered with > but a weighted coin can't, making it harder to influence how other nodes > handle the request. A lot more requests will naturally timeout than do now, and therefore a node can silently drop a request without arousing much concern or suspicion. Per node failure tables can't reroute that key until the timeout happens, and if the timeout is larger than it is now, that takes longer. > > > - The current counter is reset when we get closer to the target, so to some > > degree it adapts to the network size. We'd lose this, so we may have to set > > the probability of termination quite low, and fiddle with it... > > Good point, I didn't realise that resetting the counter had that function. > > > Why can we not analyse how much information the current counter leaks? Because > > of the uncertain topology/average number of resets? > > I shouldn't have said we can't analyse it, but I wouldn't know where to > start because so much depends on the topology. For example, the > closest-location-so-far obviously leaks information, but how do we > quantify it? Isn't it similar to the situation with keys? -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/c21df9d9/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 00:08:57 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 00:08:57 +0000 Subject: [freenet-dev] Tunnels vs premix routing was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <47779AEF.6080305@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200712210025.29457.toad@amphibian.dyndns.org> <47779AEF.6080305@cs.ucl.ac.uk> Message-ID: <200801030008.57919.toad@amphibian.dyndns.org> On Sunday 30 December 2007 13:19, Michael Rogers wrote: > > >> This could actually be used in place of premix routing: instead of > >> constructing cells we use random "tunnel IDs". > > > > No use against a local attacker. > > What kind of attack do you have in mind? "Local" = direct peer. If we use one tunnel per local pseudo-identity, it might work, but there is still a very high chance that the node sending the request is the originator. OTOH if we use more than one tunnel, either local collusion or rerouting may make life *much* easier for the attacker. > > > So a local attacker *immediately* knows that the keys are connected. > > Right, but who cares if the attacker knows that? The attacker doesn't > know whether you're the initiator of the tunnel. Makes it easier to track down the origin node for a swarm of requests once it reaches the request stage. Maybe that's not such a big deal. > > > This has been proposed before. One weakness is you'd need more than one tunnel > > to get good or predictable performance. And as soon as you have multiple > > correlatable or identical tunnels from the same node, it gets really easy to > > identify the requestor - admittedly it may require a conspiracy... > > The number of tunnels should be kept to a minimum to slow down > predecessor attacks (that's I2P's major weakness: tunnels are rebuilt > too frequently). 100 tunnels will give an attacker as much information > as 100 requests under the current system. > > On the other hand if you use separate tunnels for separate files then > it's harder for an attacker to link them, which makes it harder to use > them in a predecessor attack. So the initiator should choose tunnels > intelligently - eg give each client its own tunnel and each large > splitfile its own tunnel. For performance most users would likely want the node to use multiple tunnels for a single splitfile, and arguably this is a security issue: if the attacker receives the tunnel, and doesn't like its contents, he can trickle-feed it as an effective DoS. Note that an attacker in this case can be any node along the entire tunnel path, because they all know the keys. For premix routing, any effective attack requires that the tunnel exit node is owned by the attacker, and usually at least one node prior to that also. > > > Also, you have a very small anonymity set: if the tunnel is on average 5 hops > > long, there is a 20% chance that the node that sent you the request is the > > originator. This is unacceptable, especially if you can correlate the 20%'s > > because of linked but non-identical tunnels. > > True, but the same applies to premix routing: even if there are 100 > nodes in the cell, if you premix for 5 hops there's a 20% chance the > previous hop is the initiator. Misleading. :) If you are the exit node, you know there is a 1/cellSize chance the previous hop is the initiator. You can only know there is a high chance if you control both the exit node and a nonadjacent node before that, and you can tell that both are on the same chain. IMHO we can eliminate covert signalling, and without that all you have is the request timing, which may be enough, but will require a lot of requests to be sure of anything. Much superior to tunneling IMHO. > > > Finally, are there any vulnerabilities in changing paths? > > Yes - churn will force initiators to build new tunnels, speeding up the > predecessor attack. > > > Indeed, random hops seem to be the way forward. But if you get the request > > during the random hop phase, you can attack the originator, although it will > > take much more time than without random hops. > > Yup, there's no way to prevent predecessor attacks as far as I can tell, > but we can slow them down to one sample per tunnel instead of one sample > per request. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/795d78e5/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 00:11:55 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 00:11:55 +0000 Subject: [freenet-dev] Vulnerability in inserting the manifest before finishing In-Reply-To: <47779CB4.2050000@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200712210139.35258.toad@amphibian.dyndns.org> <47779CB4.2050000@cs.ucl.ac.uk> Message-ID: <200801030011.55484.toad@amphibian.dyndns.org> On Sunday 30 December 2007 13:27, Michael Rogers wrote: > Matthew Toseland wrote: > > If you route randomly on each hop I'd > > expect to on average not get very far, because *most links are short links* > > on a small-world network. > > Yup, it's counter-intuitive because of clustering but small world > networks do have fast mixing times, ie short random walks will quickly > take you out of your local neighbourhood. Interesting. > > > On the other hand if you choose a random location > > and consistently route towards it you should have an equal chance of ending > > up anywhere on the network. > > Almost equal - the probability of ending up at a node is proportional to > the amount of keyspace it controls, which is somewhat unevenly > distributed even in an ideal network. Sure. > > > The catch is that if you route lots of requests > > to different random locations, and the attacker can connect the requests, he > > can gradually narrow down the locations you could have started in. > > True, and Borisov doesn't address this problem (which IMO is a major > one). As you know, my proposed solution is to reuse the same random path > for as long as possible (tunnels). Right. Despite my other posts, I'm not totally against tunnels, perhaps after premix routing. But IMHO we will need quite a few of them for good performance / reliability, even several for a single request group. > > > On a cell of 10,000 nodes, if on > > average each node goes down once a day, and this impacts on 10 other nodes, > > we have ~ 100,000 messages ... perhaps hundreds of megabytes, so it may be > > that cells this big are feasible. > > There's a tradeoff here: large cells provide a large anonymity set if > the attacker's outside the cell, but they make it more likely that > there's an attacker inside the cell who can attack key distribution etc. Sure, but inside the cell it's harder to attack than outside it, surely that is the whole point? Tor, I2P, Mixmaster and other traditional onion routers (apart from spanning tree impl's) use a whole-network cell. > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/4fdda461/attachment.pgp From robert at emu.freenetproject.org Thu Jan 3 00:23:28 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Wed, 2 Jan 2008 18:23:28 -0600 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <200801022144.58447.toad@amphibian.dyndns.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801022144.58447.toad@amphibian.dyndns.org> Message-ID: <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> On Jan 2, 2008, at 3:44 PM, Matthew Toseland wrote: > On Wednesday 02 January 2008 17:22, Robert Hailey wrote: >> >> On Dec 29, 2007, at 5:39 AM, Florent Daigni?re wrote: >> >>>>> synchronized void updateShouldDisconnectNow() { >>>>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH >>>>> THE NODE >>>>> - if (isConnected()) { >>>>> + if (isConnected() || verifiedIncompatibleOlderVersion || >>>>> verifiedIncompatibleNewerVersion) { >>>>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); >>>>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); >>>>> } >>>> >>>> I suggest you call isRoutable() instead >>>> >>>> NextGen$ >>> >>> Hmm, on a latter thought, I don't understand why it helps... nor why >>> we >>> are calling it from PacketSender >>> >>> What about getting rid of both the "if" branch and the call in >>> PacketSender ? >> >> In PacketSender this function is called ~precisely~ at the version >> transition time so that all peers will be re-evaluated concerning >> compatible versioning; this seems like an odd place to stick this >> code, but also updateShouldDisconnectNow() does not explicitly >> disconnect the peer node, it only updates the 'incompatible' flags. > > We don't want to disconnect. We want to stay connected and to not > route > requests to the node or accept them from it. We will still accept > e.g. Update > Over Mandatory traffic. Fair enough, but this is only confused semantics. I said disconnect as it is in the function name, perhaps a better name would be updateVersionRoutablity(). >> Concerning the if branch I added, the intended behaviour change is >> that this function should only (1) allow the disconnection of a >> connected peer by setting the incompatible flag, > > No, we don't want to disconnect. We just want to make it non- > routable, by > updating verified*. I used the wrong term. It will only update the verified* booleans (and thus make routable or not) if connected. But see below... >> Now, if the nodes can handshake, this code has probably has little >> effect, as it simply further restricts one of only two places that >> the >> incompatible flags are set (the other being on a handshake). What it >> will do is if the nodes can't talk to each other (like in my re-keyed >> node case), is not show 'TOO_OLD'/'TOO_NEW', but instead >> 'DISCONNECTED'. > > So you are setting verified* to false, because it's not verified? Not setting it to false, but... not allowing it to be set to true if that node cannot reach us (e.g. disconnected). >> And I suppose in the same error case... if the node is >> restarted, it would never be able to verify that the peer is too old/ >> new and fetch the ark (keeping the version number up-to-date). > > Again, we should always fetch the ARK. The original implementation appears to use verifiedIncompatible* to optimize against a time-based trigger (Version-Incompatibility) from a bunch of ark requests being started. On Jan 2, 2008, at 12:30 PM, David Sowder wrote: > I'm not sure of the thinking of others regarding changes to the > verifiedIncompatible variables since, but my thinking in my initial > implementation was that they would only be set after a handshake, thus > we already had a connected to them and ARKs weren't needed. That > won't > change until the peer re-handshakes after restarting with a different > version of the node's software. As best I can see, the verified* booleans have changed meanings since update-over-mandatory. If it is still useful to know if the peer node is presently-verified-{older/newer}, then we should split the booleans (isOlder,isVerifiedOlder); otherwise just fetch an ark even if it is an older (as Matthew suggested). In either case the misleading comments and variable names should be removed and/or changed. -- Robert Hailey From freenet-devl at david.sowder.com Thu Jan 3 00:45:48 2008 From: freenet-devl at david.sowder.com (David Sowder (Zothar)) Date: Wed, 02 Jan 2008 18:45:48 -0600 Subject: [freenet-dev] [freenet-cvs] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <200801022140.35033.toad@amphibian.dyndns.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <199BD599-D8DF-40FB-975B-2BF94854BF16@emu.freenetproject.org> <477BD863.50803@david.sowder.com> <200801022140.35033.toad@amphibian.dyndns.org> Message-ID: <477C303C.3040504@david.sowder.com> In my thinking, we should only change verifiedIncompatibleOlderVersion and verifiedIncompatibleNewerVersion in four situations, all of which may even mean that we call a function/method to check for the need to change in the code paths that are common to each of these situations: 1) Potentially set when we've just completed a handshake and therefore are connected and isConnected() is true (not necessarily isRoutable() for whatever reason) after which we have no need to fetch an ARK for the peer 2) Cleared if we are disconnecting (for whatever reason) from a peer that has had either verifiedIncompatibleOlderVersion or verifiedIncompatibleNewerVersion set, after which we may want to fetch an ARK for the peer just like we would if we had never connected to them since our last restart 3) Set verifiedIncompatibleOlderVersion when our node passes a mandatory threshold date and time 4) Set verifiedIncompatibleNewerVersion when the peer tells us we are too old for their idea of the mandatory threshold date and time (I don't recall by what mechanism this is accomplished ATM) Once verifiedIncompatibleOlderVersion or verifiedIncompatibleNewerVersion is set for a peer, it cannot be cleared by any means other than the peer reconnecting (either by a intermediate DISCONNECTED, etc. state or by boot ID change (if I understand that name/concept correctly)) after their restart or by our node's initializing the value to it's default False value on our node's restart. Caveat the above by the fact that I haven't read this related code in awhile, so I'm going by my memory of how it should work and what I remember of changes made by others since. Matthew Toseland wrote: > On Wednesday 02 January 2008 18:30, David Sowder wrote: > >> I'm not sure of the thinking of others regarding changes to the >> verifiedIncompatible variables since, but my thinking in my initial >> implementation was that they would only be set after a handshake, thus >> we already had a connected to them and ARKs weren't needed. That won't >> change until the peer re-handshakes after restarting with a different >> version of the node's software. >> > > I don't follow - clearly if isConnected(), we don't need the ARK. > > ------------------------------------------------------------------------ > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From freenet-devl at david.sowder.com Thu Jan 3 00:50:37 2008 From: freenet-devl at david.sowder.com (David Sowder (Zothar)) Date: Wed, 02 Jan 2008 18:50:37 -0600 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801022144.58447.toad@amphibian.dyndns.org> <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> Message-ID: <477C315D.8010609@david.sowder.com> To be clear, a TOO OLD/TOO NEW peer is currently connected in the sense that the node is exchanging traffic with it. The node just doesn't route any traffic through the peer and the traffic with the peer is limited to keeping the connectivity, N2NMs and UoM exchanges (I don't believe I'm forgetting anything, but it wouldn't be the first time). Robert Hailey wrote: > On Jan 2, 2008, at 3:44 PM, Matthew Toseland wrote: > > >> On Wednesday 02 January 2008 17:22, Robert Hailey wrote: >> >>> On Dec 29, 2007, at 5:39 AM, Florent Daigni?re wrote: >>> >>> >>>>>> synchronized void updateShouldDisconnectNow() { >>>>>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH >>>>>> THE NODE >>>>>> - if (isConnected()) { >>>>>> + if (isConnected() || verifiedIncompatibleOlderVersion || >>>>>> verifiedIncompatibleNewerVersion) { >>>>>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); >>>>>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); >>>>>> } >>>>>> >>>>> I suggest you call isRoutable() instead >>>>> >>>>> NextGen$ >>>>> >>>> Hmm, on a latter thought, I don't understand why it helps... nor why >>>> we >>>> are calling it from PacketSender >>>> >>>> What about getting rid of both the "if" branch and the call in >>>> PacketSender ? >>>> >>> In PacketSender this function is called ~precisely~ at the version >>> transition time so that all peers will be re-evaluated concerning >>> compatible versioning; this seems like an odd place to stick this >>> code, but also updateShouldDisconnectNow() does not explicitly >>> disconnect the peer node, it only updates the 'incompatible' flags. >>> >> We don't want to disconnect. We want to stay connected and to not >> route >> requests to the node or accept them from it. We will still accept >> e.g. Update >> Over Mandatory traffic. >> > > Fair enough, but this is only confused semantics. I said disconnect as > it is in the function name, perhaps a better name would be > updateVersionRoutablity(). > > >>> Concerning the if branch I added, the intended behaviour change is >>> that this function should only (1) allow the disconnection of a >>> connected peer by setting the incompatible flag, >>> >> No, we don't want to disconnect. We just want to make it non- >> routable, by >> updating verified*. >> > > I used the wrong term. It will only update the verified* booleans (and > thus make routable or not) if connected. But see below... > > >>> Now, if the nodes can handshake, this code has probably has little >>> effect, as it simply further restricts one of only two places that >>> the >>> incompatible flags are set (the other being on a handshake). What it >>> will do is if the nodes can't talk to each other (like in my re-keyed >>> node case), is not show 'TOO_OLD'/'TOO_NEW', but instead >>> 'DISCONNECTED'. >>> >> So you are setting verified* to false, because it's not verified? >> > > Not setting it to false, but... not allowing it to be set to true if > that node cannot reach us (e.g. disconnected). > > >>> And I suppose in the same error case... if the node is >>> restarted, it would never be able to verify that the peer is too old/ >>> new and fetch the ark (keeping the version number up-to-date). >>> >> Again, we should always fetch the ARK. >> > > The original implementation appears to use verifiedIncompatible* to > optimize against a time-based trigger (Version-Incompatibility) from a > bunch of ark requests being started. > > On Jan 2, 2008, at 12:30 PM, David Sowder wrote: > >> I'm not sure of the thinking of others regarding changes to the >> verifiedIncompatible variables since, but my thinking in my initial >> implementation was that they would only be set after a handshake, thus >> we already had a connected to them and ARKs weren't needed. That >> won't >> change until the peer re-handshakes after restarting with a different >> version of the node's software. >> > > As best I can see, the verified* booleans have changed meanings since > update-over-mandatory. If it is still useful to know if the peer node > is presently-verified-{older/newer}, then we should split the booleans > (isOlder,isVerifiedOlder); otherwise just fetch an ark even if it is > an older (as Matthew suggested). In either case the misleading > comments and variable names should be removed and/or changed. > > -- > Robert Hailey > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > From m.rogers at cs.ucl.ac.uk Thu Jan 3 01:17:19 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Thu, 03 Jan 2008 01:17:19 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <200801022352.54143.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200712210025.29457.toad@amphibian.dyndns.org> <47779AEF.6080305@cs.ucl.ac.uk> <200801022352.54143.toad@amphibian.dyndns.org> Message-ID: <477C379F.4050706@cs.ucl.ac.uk> Matthew Toseland wrote: > - It should be possible with such a simple request/response to make it > extremely difficult for two nodes to realise they are on the same path, > except in the case where they are actually adjacent. I think that's optimistic - they could use the tunnel construction time, or the delay between the tunnel being constructed and used, or (for two-way tunnels) the delay between the request and the response. Let's say the attacker controls the exit node of tunnel X, and it normally takes 5 seconds to construct a tunnel. If an interesting request is sent through tunnel X, all the attacker's nodes record predecessor samples for any outgoing tunnels constructed in the 5 seconds before tunnel X was constructed, and successor samples for any incoming tunnels constructed in the 5 seconds after tunnel X was constructed. It doesn't matter that most of the samples are wrong - all that matters is that the initiator is more likely to be recorded than a random node (or a random member of the cell). If the attacker knows that tunnels always contain more than two nodes and don't pass through any node more than once, he can also eliminate samples from the exit node and its immediate neighbours. But that's not vital, it just reduces the level of noise. > However, it *is* > known that it is one of the nodes along that path that is the problem, and > this may be enough. Yup, if we can trust the topological information then we should be able to identify unreliable nodes and avoid using them in tunnels (but on the other hand that allows an attacker to attract more tunnels and thus gather more samples by performing well, as in I2P). > Each node publishes its links, including its pubkeys, their pubkey hashes, and > a signature. This is relayed to some distance. We compute a credibility value > for each node on the graph by starting with a value of 1.0 on each node we > are a direct peer of, and dividing it by the number of peers on each node > going outwards, adding up the values from different branches. We can then > ignore nodes which have a low credibility value. Sounds promising, but the result will be different from each node's point of view. How do we ensure that all members of the cell agree about which nodes to ignore? If they don't agree then it's easy for an attacker to calculate credibility values from every other node's point of view in order to work out who would or wouldn't have chosen a given exit node. > Now, we may not be able to > do exactly this, if we have to have semi-permanent cell structures, but it > should be possible to find an algorithm to serve this purpose (eigenvector > centrality? TrustRank etc?). We're caught in a cleft stick here: algorithms that give a universally agreed score for each node are vulnerable to Sybil attacks [1], but if a node's score depends on who evaluates it (eg the flow-based algorithm you sketched above) then how do we agree about which nodes to ignore? Cheers, Michael [1] http://www.sigcomm.org/sigcomm2005/paper-CheFri.pdf From m.rogers at cs.ucl.ac.uk Thu Jan 3 01:35:43 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Thu, 03 Jan 2008 01:35:43 +0000 Subject: [freenet-dev] Why getting rid of HTL is bad was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <200801030000.13355.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200712210025.29457.toad@amphibian.dyndns.org> <47779AEF.6080305@cs.ucl.ac.uk> <200801030000.13355.toad@amphibian.dyndns.org> Message-ID: <477C3BEF.9090308@cs.ucl.ac.uk> Matthew Toseland wrote: > No, but it would make the request time variance much higher, especially for > inserts and unsuccessful requests (which can potentially visit the entire > network). True, the variance would be higher. Visiting the entire network is a bit of an extreme example though - in theory it's also possible with the current hop counter, but it's not exactly likely. :-) > A lot more requests will naturally timeout than do now, and therefore a node > can silently drop a request without arousing much concern or suspicion. Per > node failure tables can't reroute that key until the timeout happens, and if > the timeout is larger than it is now, that takes longer. We're using a reliable transport between nodes, so the only things that should cause a timeout are overloaded, crashed or faulty nodes. If we can keep the ping time under control then we can detect crashed nodes quickly. So with proper load management we should be able to pretty much eliminate timeouts except in the case of faulty nodes, which can then be detected. >> I shouldn't have said we can't analyse it, but I wouldn't know where to >> start because so much depends on the topology. For example, the >> closest-location-so-far obviously leaks information, but how do we >> quantify it? > > Isn't it similar to the situation with keys? Sorry, I don't follow. The closest-location-so-far tells an attacker something about the path a request has followed - among other things, it allows the attacker to rule out certain nodes the request definitely hasn't passed through. But how would we quantify that in terms of the size of the initiator's anonymity set or the probability that a given node is the initiator, for example? Whereas with a weighted coin, if the probability that each node terminates the request is 1/n then there's a 1/n chance that the previous hop is the initiator. Cheers, Michael From m.rogers at cs.ucl.ac.uk Thu Jan 3 01:49:20 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Thu, 03 Jan 2008 01:49:20 +0000 Subject: [freenet-dev] Tunnels vs premix routing was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <200801030008.57919.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200712210025.29457.toad@amphibian.dyndns.org> <47779AEF.6080305@cs.ucl.ac.uk> <200801030008.57919.toad@amphibian.dyndns.org> Message-ID: <477C3F20.7070208@cs.ucl.ac.uk> Matthew Toseland wrote: > "Local" = direct peer. If we use one tunnel per local pseudo-identity, it > might work, but there is still a very high chance that the node sending the > request is the originator. Right, but that's also the case at the moment: if requests travel n hops on average then there's a 1/n chance that the previous hop is the initiator. I don't see how you can get away from that without onion routing. > OTOH if we use more than one tunnel, either local > collusion or rerouting may make life *much* easier for the attacker. Yes, we should keep the number of linkable tunnels to a minimum. > Makes it easier to track down the origin node for a swarm of requests once it > reaches the request stage. Maybe that's not such a big deal. If tunnels do their job then it doesn't matter if an attacker can find the last node in the tunnel. > For performance most users would likely want the node to use multiple tunnels > for a single splitfile, and arguably this is a security issue: if the > attacker receives the tunnel, and doesn't like its contents, he can > trickle-feed it as an effective DoS. Users who value performance over anonymity could use multiple tunnels per splitfile and/or shorter tunnels. I'm just saying that for maximum anonymity, the tunnels should be long enough that any node could be the initiator, and the number of linkable tunnels should be minimised (eg one tunnel per Frost ID per session rather than one per message). > For > premix routing, any effective attack requires that the tunnel exit node is > owned by the attacker, and usually at least one node prior to that also. True, I'm not suggesting that tunnels are as strong as premix routing, but on the other hand they don't require revealing the topology or distributing (and agreeing on) public keys for every node in the cell. Cheers, Michael From m.rogers at cs.ucl.ac.uk Thu Jan 3 02:07:19 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Thu, 03 Jan 2008 02:07:19 +0000 Subject: [freenet-dev] Vulnerability in inserting the manifest before finishing In-Reply-To: <200801030011.55484.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200712210139.35258.toad@amphibian.dyndns.org> <47779CB4.2050000@cs.ucl.ac.uk> <200801030011.55484.toad@amphibian.dyndns.org> Message-ID: <477C4357.6020805@cs.ucl.ac.uk> Matthew Toseland wrote: >> There's a tradeoff here: large cells provide a large anonymity set if >> the attacker's outside the cell, but they make it more likely that >> there's an attacker inside the cell who can attack key distribution etc. > > Sure, but inside the cell it's harder to attack than outside it, surely that > is the whole point? Tor, I2P, Mixmaster and other traditional onion routers > (apart from spanning tree impl's) use a whole-network cell. Right, but they all fudge the key distribution problem. Tor and Mixminion use central directory servers; I2P uses a DHT made of untrusted nodes; Tarzan uses an insecure and unscalable gossip protocol; MorphMix uses an insecure collusion detection mechanism [1]. I understand why premix routing is preferable to tunnels but you have to solve the key distribution problem - your credibility score proposal sounds promising but I'm not sure it's possible to make it Sybilproof and still ensure that all nodes agree about the topology. Cheers, Michael [1] http://www.crhc.uiuc.edu/~nikita/papers/pet2006-morphmix.pdf From nextgens at freenetproject.org Thu Jan 3 10:18:37 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Thu, 3 Jan 2008 11:18:37 +0100 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801022144.58447.toad@amphibian.dyndns.org> <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> Message-ID: <20080103101836.GA4339@freenetproject.org> * Robert Hailey [2008-01-02 18:23:28]: > > On Jan 2, 2008, at 3:44 PM, Matthew Toseland wrote: > > > On Wednesday 02 January 2008 17:22, Robert Hailey wrote: > >> > >> On Dec 29, 2007, at 5:39 AM, Florent Daigni?re wrote: > >> > >>>>> synchronized void updateShouldDisconnectNow() { > >>>>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH > >>>>> THE NODE > >>>>> - if (isConnected()) { > >>>>> + if (isConnected() || verifiedIncompatibleOlderVersion || > >>>>> verifiedIncompatibleNewerVersion) { > >>>>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); > >>>>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); > >>>>> } > >>>> > >>>> I suggest you call isRoutable() instead > >>>> > >>>> NextGen$ > >>> > >>> Hmm, on a latter thought, I don't understand why it helps... nor why > >>> we > >>> are calling it from PacketSender > >>> > >>> What about getting rid of both the "if" branch and the call in > >>> PacketSender ? > >> > >> In PacketSender this function is called ~precisely~ at the version > >> transition time so that all peers will be re-evaluated concerning > >> compatible versioning; this seems like an odd place to stick this > >> code, but also updateShouldDisconnectNow() does not explicitly > >> disconnect the peer node, it only updates the 'incompatible' flags. > > > > We don't want to disconnect. We want to stay connected and to not > > route > > requests to the node or accept them from it. We will still accept > > e.g. Update > > Over Mandatory traffic. > > Fair enough, but this is only confused semantics. I said disconnect as > it is in the function name, perhaps a better name would be > updateVersionRoutablity(). > > >> Concerning the if branch I added, the intended behaviour change is > >> that this function should only (1) allow the disconnection of a > >> connected peer by setting the incompatible flag, > > > > No, we don't want to disconnect. We just want to make it non- > > routable, by > > updating verified*. > > I used the wrong term. It will only update the verified* booleans (and > thus make routable or not) if connected. But see below... > > >> Now, if the nodes can handshake, this code has probably has little > >> effect, as it simply further restricts one of only two places that > >> the > >> incompatible flags are set (the other being on a handshake). What it > >> will do is if the nodes can't talk to each other (like in my re-keyed > >> node case), is not show 'TOO_OLD'/'TOO_NEW', but instead > >> 'DISCONNECTED'. > > > > So you are setting verified* to false, because it's not verified? > > Not setting it to false, but... not allowing it to be set to true if > that node cannot reach us (e.g. disconnected). > > >> And I suppose in the same error case... if the node is > >> restarted, it would never be able to verify that the peer is too old/ > >> new and fetch the ark (keeping the version number up-to-date). > > > > Again, we should always fetch the ARK. > > The original implementation appears to use verifiedIncompatible* to > optimize against a time-based trigger (Version-Incompatibility) from a > bunch of ark requests being started. > > On Jan 2, 2008, at 12:30 PM, David Sowder wrote: > > I'm not sure of the thinking of others regarding changes to the > > verifiedIncompatible variables since, but my thinking in my initial > > implementation was that they would only be set after a handshake, thus > > we already had a connected to them and ARKs weren't needed. That > > won't > > change until the peer re-handshakes after restarting with a different > > version of the node's software. > > As best I can see, the verified* booleans have changed meanings since > update-over-mandatory. If it is still useful to know if the peer node > is presently-verified-{older/newer}, then we should split the booleans > (isOlder,isVerifiedOlder); otherwise just fetch an ark even if it is > an older (as Matthew suggested). In either case the misleading > comments and variable names should be removed and/or changed. We want to fetch ARKs in any case imo. NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/0a0bd17f/attachment.pgp From freenet-devl at david.sowder.com Thu Jan 3 13:36:39 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Thu, 03 Jan 2008 07:36:39 -0600 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <20080103101836.GA4339@freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801022144.58447.toad@amphibian.dyndns.org> <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> <20080103101836.GA4339@freenetproject.org> Message-ID: <477CE4E7.5020700@david.sowder.com> Florent Daigni?re wrote: > * Robert Hailey [2008-01-02 18:23:28]: > > >> On Jan 2, 2008, at 3:44 PM, Matthew Toseland wrote: >> >> >>> On Wednesday 02 January 2008 17:22, Robert Hailey wrote: >>> >>>> On Dec 29, 2007, at 5:39 AM, Florent Daigni?re wrote: >>>> >>>> >>>>>>> synchronized void updateShouldDisconnectNow() { >>>>>>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH >>>>>>> THE NODE >>>>>>> - if (isConnected()) { >>>>>>> + if (isConnected() || verifiedIncompatibleOlderVersion || >>>>>>> verifiedIncompatibleNewerVersion) { >>>>>>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); >>>>>>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); >>>>>>> } >>>>>>> >>>>>> I suggest you call isRoutable() instead >>>>>> >>>>>> NextGen$ >>>>>> >>>>> Hmm, on a latter thought, I don't understand why it helps... nor why >>>>> we >>>>> are calling it from PacketSender >>>>> >>>>> What about getting rid of both the "if" branch and the call in >>>>> PacketSender ? >>>>> >>>> In PacketSender this function is called ~precisely~ at the version >>>> transition time so that all peers will be re-evaluated concerning >>>> compatible versioning; this seems like an odd place to stick this >>>> code, but also updateShouldDisconnectNow() does not explicitly >>>> disconnect the peer node, it only updates the 'incompatible' flags. >>>> >>> We don't want to disconnect. We want to stay connected and to not >>> route >>> requests to the node or accept them from it. We will still accept >>> e.g. Update >>> Over Mandatory traffic. >>> >> Fair enough, but this is only confused semantics. I said disconnect as >> it is in the function name, perhaps a better name would be >> updateVersionRoutablity(). >> >> >>>> Concerning the if branch I added, the intended behaviour change is >>>> that this function should only (1) allow the disconnection of a >>>> connected peer by setting the incompatible flag, >>>> >>> No, we don't want to disconnect. We just want to make it non- >>> routable, by >>> updating verified*. >>> >> I used the wrong term. It will only update the verified* booleans (and >> thus make routable or not) if connected. But see below... >> >> >>>> Now, if the nodes can handshake, this code has probably has little >>>> effect, as it simply further restricts one of only two places that >>>> the >>>> incompatible flags are set (the other being on a handshake). What it >>>> will do is if the nodes can't talk to each other (like in my re-keyed >>>> node case), is not show 'TOO_OLD'/'TOO_NEW', but instead >>>> 'DISCONNECTED'. >>>> >>> So you are setting verified* to false, because it's not verified? >>> >> Not setting it to false, but... not allowing it to be set to true if >> that node cannot reach us (e.g. disconnected). >> >> >>>> And I suppose in the same error case... if the node is >>>> restarted, it would never be able to verify that the peer is too old/ >>>> new and fetch the ark (keeping the version number up-to-date). >>>> >>> Again, we should always fetch the ARK. >>> >> The original implementation appears to use verifiedIncompatible* to >> optimize against a time-based trigger (Version-Incompatibility) from a >> bunch of ark requests being started. >> >> On Jan 2, 2008, at 12:30 PM, David Sowder wrote: >> >>> I'm not sure of the thinking of others regarding changes to the >>> verifiedIncompatible variables since, but my thinking in my initial >>> implementation was that they would only be set after a handshake, thus >>> we already had a connected to them and ARKs weren't needed. That >>> won't >>> change until the peer re-handshakes after restarting with a different >>> version of the node's software. >>> >> As best I can see, the verified* booleans have changed meanings since >> update-over-mandatory. If it is still useful to know if the peer node >> is presently-verified-{older/newer}, then we should split the booleans >> (isOlder,isVerifiedOlder); otherwise just fetch an ark even if it is >> an older (as Matthew suggested). In either case the misleading >> comments and variable names should be removed and/or changed. >> > > We want to fetch ARKs in any case imo. > And in my opinion, fetching an ARK for a connected peer is pointless, but perhaps we agree in our assessments and I misunderstand by thinking you mean to _always_ fetch ARKs regardless of current peer status. From nextgens at freenetproject.org Thu Jan 3 13:44:33 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Thu, 3 Jan 2008 14:44:33 +0100 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <477CE4E7.5020700@david.sowder.com> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801022144.58447.toad@amphibian.dyndns.org> <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> <20080103101836.GA4339@freenetproject.org> <477CE4E7.5020700@david.sowder.com> Message-ID: <20080103134433.GC4339@freenetproject.org> * David Sowder [2008-01-03 07:36:39]: > Florent Daigni?re wrote: >> * Robert Hailey [2008-01-02 18:23:28]: >> >> >>> On Jan 2, 2008, at 3:44 PM, Matthew Toseland wrote: >>> >>> >>>> On Wednesday 02 January 2008 17:22, Robert Hailey wrote: >>>> >>>>> On Dec 29, 2007, at 5:39 AM, Florent Daigni?re wrote: >>>>> >>>>> >>>>>>>> synchronized void updateShouldDisconnectNow() { >>>>>>>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH >>>>>>>> THE NODE >>>>>>>> - if (isConnected()) { >>>>>>>> + if (isConnected() || verifiedIncompatibleOlderVersion || >>>>>>>> verifiedIncompatibleNewerVersion) { >>>>>>>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); >>>>>>>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); >>>>>>>> } >>>>>>>> >>>>>>> I suggest you call isRoutable() instead >>>>>>> >>>>>>> NextGen$ >>>>>>> >>>>>> Hmm, on a latter thought, I don't understand why it helps... nor why >>>>>> we >>>>>> are calling it from PacketSender >>>>>> >>>>>> What about getting rid of both the "if" branch and the call in >>>>>> PacketSender ? >>>>>> >>>>> In PacketSender this function is called ~precisely~ at the version >>>>> transition time so that all peers will be re-evaluated concerning >>>>> compatible versioning; this seems like an odd place to stick this >>>>> code, but also updateShouldDisconnectNow() does not explicitly >>>>> disconnect the peer node, it only updates the 'incompatible' flags. >>>>> >>>> We don't want to disconnect. We want to stay connected and to not route >>>> requests to the node or accept them from it. We will still accept e.g. >>>> Update >>>> Over Mandatory traffic. >>>> >>> Fair enough, but this is only confused semantics. I said disconnect as >>> it is in the function name, perhaps a better name would be >>> updateVersionRoutablity(). >>> >>> >>>>> Concerning the if branch I added, the intended behaviour change is >>>>> that this function should only (1) allow the disconnection of a >>>>> connected peer by setting the incompatible flag, >>>>> >>>> No, we don't want to disconnect. We just want to make it non- routable, >>>> by >>>> updating verified*. >>>> >>> I used the wrong term. It will only update the verified* booleans (and >>> thus make routable or not) if connected. But see below... >>> >>> >>>>> Now, if the nodes can handshake, this code has probably has little >>>>> effect, as it simply further restricts one of only two places that the >>>>> incompatible flags are set (the other being on a handshake). What it >>>>> will do is if the nodes can't talk to each other (like in my re-keyed >>>>> node case), is not show 'TOO_OLD'/'TOO_NEW', but instead >>>>> 'DISCONNECTED'. >>>>> >>>> So you are setting verified* to false, because it's not verified? >>>> >>> Not setting it to false, but... not allowing it to be set to true if >>> that node cannot reach us (e.g. disconnected). >>> >>> >>>>> And I suppose in the same error case... if the node is >>>>> restarted, it would never be able to verify that the peer is too old/ >>>>> new and fetch the ark (keeping the version number up-to-date). >>>>> >>>> Again, we should always fetch the ARK. >>>> >>> The original implementation appears to use verifiedIncompatible* to >>> optimize against a time-based trigger (Version-Incompatibility) from a >>> bunch of ark requests being started. >>> >>> On Jan 2, 2008, at 12:30 PM, David Sowder wrote: >>> >>>> I'm not sure of the thinking of others regarding changes to the >>>> verifiedIncompatible variables since, but my thinking in my initial >>>> implementation was that they would only be set after a handshake, thus >>>> we already had a connected to them and ARKs weren't needed. That won't >>>> change until the peer re-handshakes after restarting with a different >>>> version of the node's software. >>>> >>> As best I can see, the verified* booleans have changed meanings since >>> update-over-mandatory. If it is still useful to know if the peer node is >>> presently-verified-{older/newer}, then we should split the booleans >>> (isOlder,isVerifiedOlder); otherwise just fetch an ark even if it is an >>> older (as Matthew suggested). In either case the misleading comments and >>> variable names should be removed and/or changed. >>> >> >> We want to fetch ARKs in any case imo. >> > And in my opinion, fetching an ARK for a connected peer is pointless, but > perhaps we agree in our assessments and I misunderstand by thinking you > mean to _always_ fetch ARKs regardless of current peer status. It's not pointless. We want to keep an up to date "edition" for the ark so that when we will need it we will be able to reach it... Of course we could exchange and update the edition number on handshake... but atm we don't. NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/851bc578/attachment.pgp From freenet-devl at david.sowder.com Thu Jan 3 13:52:44 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Thu, 03 Jan 2008 07:52:44 -0600 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <20080103134433.GC4339@freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801022144.58447.toad@amphibian.dyndns.org> <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> <20080103101836.GA4339@freenetproject.org> <477CE4E7.5020700@david.sowder.com> <20080103134433.GC4339@freenetproject.org> Message-ID: <477CE8AC.9080307@david.sowder.com> Florent Daigni?re wrote: > * David Sowder [2008-01-03 07:36:39]: > > >> Florent Daigni?re wrote: >> >>> * Robert Hailey [2008-01-02 18:23:28]: >>> >>>> >>>> On Jan 2, 2008, at 12:30 PM, David Sowder wrote: >>>> >>>> >>>>> I'm not sure of the thinking of others regarding changes to the >>>>> verifiedIncompatible variables since, but my thinking in my initial >>>>> implementation was that they would only be set after a handshake, thus >>>>> we already had a connected to them and ARKs weren't needed. That won't >>>>> change until the peer re-handshakes after restarting with a different >>>>> version of the node's software. >>>>> >>>>> >>>> As best I can see, the verified* booleans have changed meanings since >>>> update-over-mandatory. If it is still useful to know if the peer node is >>>> presently-verified-{older/newer}, then we should split the booleans >>>> (isOlder,isVerifiedOlder); otherwise just fetch an ark even if it is an >>>> older (as Matthew suggested). In either case the misleading comments and >>>> variable names should be removed and/or changed. >>>> >>>> >>> We want to fetch ARKs in any case imo. >>> >>> >> And in my opinion, fetching an ARK for a connected peer is pointless, but >> perhaps we agree in our assessments and I misunderstand by thinking you >> mean to _always_ fetch ARKs regardless of current peer status. >> > > It's not pointless. We want to keep an up to date "edition" for the ark so > that when we will need it we will be able to reach it... Of course we > could exchange and update the edition number on handshake... but atm we > don't. > It is pointless. If the problem is that we don't have updated ARK edition numbers, then we should fix that by exchanging that across an existing connection when we have one rather than forcing the use of a backwards and network load causing method for for information we have from a directly connected peer. Fixing that is simply a matter of exchanging either full or differential references with our peers any time our node's reference changes. I'm not exactly available to implement it right now, but I know for a fact that this is not a hard problem to solve. In fact, differential references could be trivially implemented using the existing N2NM messaging facilities. From nextgens at freenetproject.org Thu Jan 3 13:57:24 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Thu, 3 Jan 2008 14:57:24 +0100 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <477CE8AC.9080307@david.sowder.com> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801022144.58447.toad@amphibian.dyndns.org> <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> <20080103101836.GA4339@freenetproject.org> <477CE4E7.5020700@david.sowder.com> <20080103134433.GC4339@freenetproject.org> <477CE8AC.9080307@david.sowder.com> Message-ID: <20080103135723.GD4339@freenetproject.org> * David Sowder [2008-01-03 07:52:44]: > Florent Daigni?re wrote: >> * David Sowder [2008-01-03 07:36:39]: >> >> >>> Florent Daigni?re wrote: >>> >>>> * Robert Hailey [2008-01-02 18:23:28]: >>>> >>>>> >>>>> On Jan 2, 2008, at 12:30 PM, David Sowder wrote: >>>>> >>>>>> I'm not sure of the thinking of others regarding changes to the >>>>>> verifiedIncompatible variables since, but my thinking in my initial >>>>>> implementation was that they would only be set after a handshake, thus >>>>>> we already had a connected to them and ARKs weren't needed. That won't >>>>>> change until the peer re-handshakes after restarting with a different >>>>>> version of the node's software. >>>>>> >>>>> As best I can see, the verified* booleans have changed meanings since >>>>> update-over-mandatory. If it is still useful to know if the peer node >>>>> is presently-verified-{older/newer}, then we should split the booleans >>>>> (isOlder,isVerifiedOlder); otherwise just fetch an ark even if it is >>>>> an older (as Matthew suggested). In either case the misleading >>>>> comments and variable names should be removed and/or changed. >>>>> >>>> We want to fetch ARKs in any case imo. >>>> >>> And in my opinion, fetching an ARK for a connected peer is pointless, but >>> perhaps we agree in our assessments and I misunderstand by thinking you >>> mean to _always_ fetch ARKs regardless of current peer status. >>> >> >> It's not pointless. We want to keep an up to date "edition" for the ark so >> that when we will need it we will be able to reach it... Of course we >> could exchange and update the edition number on handshake... but atm we >> don't. >> > It is pointless. If the problem is that we don't have updated ARK edition > numbers, then we should fix that by exchanging that across an existing > connection when we have one rather than forcing the use of a backwards and > network load causing method for for information we have from a directly > connected peer. It has at least two assets : 1) code simplicity 2) it propagates the ARK > Fixing that is simply a matter of exchanging either full > or differential references with our peers any time our node's reference > changes. Lots of things are that simple and in need of doing because noone is available to implement them. > I'm not exactly available to implement it right now, but I know > for a fact that this is not a hard problem to solve. Neither am I > In fact, differential > references could be trivially implemented using the existing N2NM messaging > facilities. Creating a new DMT message type or extending an existing one is the way to go. NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/b219de47/attachment.pgp From freenet-devl at david.sowder.com Thu Jan 3 14:33:00 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Thu, 03 Jan 2008 08:33:00 -0600 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <20080103135723.GD4339@freenetproject.org> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20071229113933.GD4199@freenetproject.org> <200801022144.58447.toad@amphibian.dyndns.org> <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> <20080103101836.GA4339@freenetproject.org> <477CE4E7.5020700@david.sowder.com> <20080103134433.GC4339@freenetproject.org> <477CE8AC.9080307@david.sowder.com> <20080103135723.GD4339@freenetproject.org> Message-ID: <477CF21C.7000301@david.sowder.com> Florent Daigni?re wrote: > * David Sowder [2008-01-03 07:52:44]: > > >> Florent Daigni?re wrote: >> >>> * David Sowder [2008-01-03 07:36:39]: >>> >>> >>> >>>> Florent Daigni?re wrote: >>>> >>>> >>>>> * Robert Hailey [2008-01-02 18:23:28]: >>>>> >>>>> >>>>>> On Jan 2, 2008, at 12:30 PM, David Sowder wrote: >>>>>> >>>>>> >>>>>>> I'm not sure of the thinking of others regarding changes to the >>>>>>> verifiedIncompatible variables since, but my thinking in my initial >>>>>>> implementation was that they would only be set after a handshake, thus >>>>>>> we already had a connected to them and ARKs weren't needed. That won't >>>>>>> change until the peer re-handshakes after restarting with a different >>>>>>> version of the node's software. >>>>>>> >>>>>>> >>>>>> As best I can see, the verified* booleans have changed meanings since >>>>>> update-over-mandatory. If it is still useful to know if the peer node >>>>>> is presently-verified-{older/newer}, then we should split the booleans >>>>>> (isOlder,isVerifiedOlder); otherwise just fetch an ark even if it is >>>>>> an older (as Matthew suggested). In either case the misleading >>>>>> comments and variable names should be removed and/or changed. >>>>>> >>>>>> >>>>> We want to fetch ARKs in any case imo. >>>>> >>>>> >>>> And in my opinion, fetching an ARK for a connected peer is pointless, but >>>> perhaps we agree in our assessments and I misunderstand by thinking you >>>> mean to _always_ fetch ARKs regardless of current peer status. >>>> >>>> >>> It's not pointless. We want to keep an up to date "edition" for the ark so >>> that when we will need it we will be able to reach it... Of course we >>> could exchange and update the edition number on handshake... but atm we >>> don't. >>> >>> >> It is pointless. If the problem is that we don't have updated ARK edition >> numbers, then we should fix that by exchanging that across an existing >> connection when we have one rather than forcing the use of a backwards and >> network load causing method for for information we have from a directly >> connected peer. >> > > It has at least two assets : 1) code simplicity 2) it propagates the ARK > IMO, both 1 and 2 are outweighed by the network load of all nodes constantly fetching ARKs for all of their peers network-wide. 1) IMO, this is a flimsy argument considering that there are many parts of the Freenet code base that are much more complex than what I'm talking about would require and many of them, while perhaps they could be simpler, have to be somewhat complex by necessity. 2) Fetching the ARK on first connect and some number of minutes or hours after the peer tells us via differential references that it has changed would have a much more gentle impact on network load. It could be argued that if ARKs aren't propagating from their inserts correctly, there is a routing issue. If we're worried about the ARK falling out of the network, we could fetch it once every seven days after it's last change. >> Fixing that is simply a matter of exchanging either full >> or differential references with our peers any time our node's reference >> changes. >> > > Lots of things are that simple and in need of doing because noone is > available to implement them. > Perhaps this one interests me enough to code up if I'm in the right mood after work some day in the coming weeks. I'll probably implement it on top of N2NMs since they were designed to be generic and I see no reason not to use them as they already support sending arbitrary SFS blocks. >> I'm not exactly available to implement it right now, but I know >> for a fact that this is not a hard problem to solve. >> > > Neither am I > > >> In fact, differential >> references could be trivially implemented using the existing N2NM messaging >> facilities. >> > > Creating a new DMT message type or extending an existing one is the way > to go. > From toad at amphibian.dyndns.org Thu Jan 3 15:03:29 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 15:03:29 +0000 Subject: [freenet-dev] [freenet-cvs] r16817 - in trunk/freenet/src/freenet: node support/math In-Reply-To: <20071226205138.AA33B47A371@freenetproject.org> References: <20071226205138.AA33B47A371@freenetproject.org> Message-ID: <200801031503.34926.toad@amphibian.dyndns.org> Useful class, but I'm really not convinced it's statistically sound. You need to normalize the stored value after every operation, otherwise as it gets larger, you will get bigger jumps when values are reported. In order to do this, you probably need to extend BDRA; put some hooks in. On Wednesday 26 December 2007 20:51, robert at freenetproject.org wrote: > Author: robert > Date: 2007-12-26 20:51:38 +0000 (Wed, 26 Dec 2007) > New Revision: 16817 > > Added: > trunk/freenet/src/freenet/support/math/DecayingKeyspaceAverage.java > Modified: > trunk/freenet/src/freenet/node/Location.java > trunk/freenet/src/freenet/node/NodeStats.java > Log: > implement keyspace-aware averager > > > Modified: trunk/freenet/src/freenet/node/Location.java > =================================================================== > --- trunk/freenet/src/freenet/node/Location.java 2007-12-26 20:48:13 UTC (rev 16816) > +++ trunk/freenet/src/freenet/node/Location.java 2007-12-26 20:51:38 UTC (rev 16817) > @@ -70,6 +70,20 @@ > } > } > > + /** > + * Given an arbitrary double (not bound to [0.0, 1.0]) return the normalized double [0.0, 1.0] which would result in simple > + * wrapping/overflowing. e.g. normalize(1.0+0.5)==0.5, normalize(0.5-1.0)==0.5, normalize({0.0,1.0})=={0.0,1.0} > + * @bug: if given double has wrapped too many times, the return value may be not be precise. > + */ > + public static double normalize(double rough) { > + int intPart=(int)rough; > + double maybeNegative=rough-intPart; > + //note: maybeNegative is bound between (-1.0, 1.0) > + if (maybeNegative < 0) > + return 1.0+maybeNegative; > + return maybeNegative; > + } > + > public static boolean equals(double newLoc, double currentLocation) { > return Math.abs(newLoc - currentLocation) < Double.MIN_VALUE * 2; > } > > Modified: trunk/freenet/src/freenet/node/NodeStats.java > =================================================================== > --- trunk/freenet/src/freenet/node/NodeStats.java 2007-12-26 20:48:13 UTC (rev 16816) > +++ trunk/freenet/src/freenet/node/NodeStats.java 2007-12-26 20:51:38 UTC (rev 16817) > @@ -23,6 +23,7 @@ > import freenet.support.api.IntCallback; > import freenet.support.api.LongCallback; > import freenet.support.math.BootstrappingDecayingRunningAverage; > +import freenet.support.math.DecayingKeyspaceAverage; > import freenet.support.math.RunningAverage; > import freenet.support.math.TimeDecayingRunningAverage; > import freenet.support.math.TrivialRunningAverage; > @@ -189,11 +190,11 @@ > this.backedOffPercent = new TimeDecayingRunningAverage(0.0, 180000, 0.0, 1.0, node); > double nodeLoc=node.lm.getLocation(); > // FIXME PLEASE; (int) casts; (maxCacheKeys>MAXINT?) This value will probably end up being a small constant anyway (200?). > - this.avgCacheLocation = new BootstrappingDecayingRunningAverage(nodeLoc, 0.0, 1.0, (int)node.maxCacheKeys, null); > - this.avgStoreLocation = new BootstrappingDecayingRunningAverage(nodeLoc, 0.0, 1.0, (int)node.maxStoreKeys, null); > - this.avgCacheSuccess = new BootstrappingDecayingRunningAverage(nodeLoc, 0.0, 1.0, 10000, null); > - this.avgStoreSuccess = new BootstrappingDecayingRunningAverage(nodeLoc, 0.0, 1.0, 10000, null); > - this.avgRequestLocation = new BootstrappingDecayingRunningAverage(nodeLoc, 0.0, 1.0, 10000, null); > + this.avgCacheLocation = new DecayingKeyspaceAverage(nodeLoc, (int)node.maxCacheKeys, null); > + this.avgStoreLocation = new DecayingKeyspaceAverage(nodeLoc, (int)node.maxStoreKeys, null); > + this.avgCacheSuccess = new DecayingKeyspaceAverage(nodeLoc, 10000, null); > + this.avgStoreSuccess = new DecayingKeyspaceAverage(nodeLoc, 10000, null); > + this.avgRequestLocation = new DecayingKeyspaceAverage(nodeLoc, 10000, null); > preemptiveRejectReasons = new StringCounter(); > localPreemptiveRejectReasons = new StringCounter(); > pInstantRejectIncoming = new TimeDecayingRunningAverage(0, 60000, 0.0, 1.0, node); > > Added: trunk/freenet/src/freenet/support/math/DecayingKeyspaceAverage.java > =================================================================== > --- trunk/freenet/src/freenet/support/math/DecayingKeyspaceAverage.java (rev 0) > +++ trunk/freenet/src/freenet/support/math/DecayingKeyspaceAverage.java 2007-12-26 20:51:38 UTC (rev 16817) > @@ -0,0 +1,139 @@ > +/* This code is part of Freenet. It is distributed under the GNU General > + * Public License, version 2 (or at your option any later version). See > + * http://www.gnu.org/ for further details of the GPL. */ > +package freenet.support.math; > + > +import freenet.node.Location; > + > +import freenet.support.Logger; > +import freenet.support.SimpleFieldSet; > + > +/** > + * @author robert > + * > + * A filter on BootstrappingDecayingRunningAverage which makes it aware of the circular keyspace. > + */ > +public class DecayingKeyspaceAverage implements RunningAverage { > + /** > + 'avg' is the non-normalized average location. > + */ > + BootstrappingDecayingRunningAverage avg; > + > + //If the keyspace averager wraps more than this number of times, an exception will be thrown. > + public final static int WRAP_WARNING=1000; > + > + public DecayingKeyspaceAverage(double defaultValue, int maxReports, SimpleFieldSet fs) { > + avg=new BootstrappingDecayingRunningAverage(defaultValue, -WRAP_WARNING, WRAP_WARNING, maxReports, fs); > + } > + > + public DecayingKeyspaceAverage(BootstrappingDecayingRunningAverage a) { > + //check the max/min values? ignore them? > + avg=(BootstrappingDecayingRunningAverage)a.clone(); > + } > + > + public synchronized Object clone() { > + return new DecayingKeyspaceAverage(avg); > + } > + > + public synchronized double currentValue() { > + return Location.normalize(avg.currentValue()); > + } > + > + public synchronized void report(double d) { > + if ((d < 0.0) || (d > 1.0)) { > + //Just because we use non-normalized locations doesn't mean we can accept them. > + throw new IllegalArgumentException("Not a valid normalized key: "+d); > + } > + double superValue=avg.currentValue(); > + double thisValue=Location.normalize(superValue); > + double diff=Location.change(thisValue, d); > + double toAverage=(superValue+diff); > + /* > + To gracefully wrap around the 1.0/0.0 threshold we average over (or under) it, and simply normalize the result when reporting a currentValue > + ---example--- > + d=0.2; //being reported > + superValue=1.9; //already wrapped once, but at 0.9 > + thisValue=0.9; //the normalized value of where we are in the keyspace > + diff = +0.3; //the diff from the normalized values; Location.change(0.9, 0.2); > + avg.report(2.2);//to successfully move the average towards the closest route to the given value. > + */ > + //System.err.println("debug: "+superValue+", "+thisValue+", "+diff+", "+(superValue+diff)+", "+Location.normalize(superValue+diff)); > + if (toAverage>WRAP_WARNING || toAverage<-WRAP_WARNING) > + Logger.error(this, "DecayingKeyspaceAverage is wrapped up too many times"); > + avg.report(toAverage); > + } > + > + public synchronized double valueIfReported(double d) { > + if ((d < 0.0) || (d > 1.0)) { > + throw new IllegalArgumentException("Not a valid normalized key: "+d); > + } > + double superValue=avg.currentValue(); > + double thisValue=Location.normalize(superValue); > + double diff=Location.change(thisValue, d); > + return avg.valueIfReported(superValue+diff); > + } > + > + public synchronized long countReports() { > + return avg.countReports(); > + } > + > + public void report(long d) { > + throw new IllegalArgumentException("KeyspaceAverage does not like longs"); > + } > + > + ///@todo: make this a junit test > + public static void main(String[] args) { > + DecayingKeyspaceAverage a=new DecayingKeyspaceAverage(0.9, 10, null); > + a.report(0.9); > + for (int i=10; i!=0; i--) { > + a.report(0.2); > + System.out.println("<-0.2-- current="+a.currentValue()); > + } > + for (int i=10; i!=0; i--) { > + a.report(0.8); > + System.out.println("--0.8-> current="+a.currentValue()); > + } > + System.out.println("--- positive wrap test ---"); > + for (int wrap=3; wrap!=0; wrap--) { > + System.out.println("wrap test #"+wrap); > + for (int i=10; i!=0; i--) { > + a.report(0.25); > + System.out.println("<-0.25- current="+a.currentValue()); > + } > + for (int i=10; i!=0; i--) { > + a.report(0.5); > + System.out.println("--0.5-> current="+a.currentValue()); > + } > + for (int i=10; i!=0; i--) { > + a.report(0.75); > + System.out.println("-0.75-> current="+a.currentValue()); > + } > + for (int i=10; i!=0; i--) { > + a.report(1.0); > + System.out.println("<-1.0-- current="+a.currentValue()); > + } > + } > + System.out.println("--- negative wrap test ---"); > + a=new DecayingKeyspaceAverage(0.2, 10, null); > + a.report(0.2); > + for (int wrap=3; wrap!=0; wrap--) { > + System.out.println("negwrap test #"+wrap); > + for (int i=10; i!=0; i--) { > + a.report(0.75); > + System.out.println("-0.75-> current="+a.currentValue()); > + } > + for (int i=10; i!=0; i--) { > + a.report(0.5); > + System.out.println("<-0.5-- current="+a.currentValue()); > + } > + for (int i=10; i!=0; i--) { > + a.report(0.25); > + System.out.println("<-0.25- current="+a.currentValue()); > + } > + for (int i=10; i!=0; i--) { > + a.report(1.0); > + System.out.println("--1.0-> current="+a.currentValue()); > + } > + } > + } > +} > \ No newline at end of file > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/ff68ba22/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 15:11:36 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 15:11:36 +0000 Subject: [freenet-dev] [freenet-cvs] r16826 - trunk/freenet/src/freenet/node In-Reply-To: <20071228165118.B96914798DD@freenetproject.org> References: <20071228165118.B96914798DD@freenetproject.org> Message-ID: <200801031511.36969.toad@amphibian.dyndns.org> Isn't sendSync() a longer timeout than the waitFor()? We could change that though, pass in a timeout..? On Friday 28 December 2007 16:51, robert at freenetproject.org wrote: > Author: robert > Date: 2007-12-28 16:51:18 +0000 (Fri, 28 Dec 2007) > New Revision: 16826 > > Modified: > trunk/freenet/src/freenet/node/InsertHandler.java > Log: > less timeouts which are really disconnects > > > Modified: trunk/freenet/src/freenet/node/InsertHandler.java > =================================================================== > --- trunk/freenet/src/freenet/node/InsertHandler.java 2007-12-28 16:50:28 UTC (rev 16825) > +++ trunk/freenet/src/freenet/node/InsertHandler.java 2007-12-28 16:51:18 UTC (rev 16826) > @@ -91,7 +91,8 @@ > // Send Accepted > Message accepted = DMT.createFNPAccepted(uid); > try { > - source.sendAsync(accepted, null, 0, this); > + //Using sendSync here will help the next message filter not timeout... wait here or at the message filter. > + source.sendSync(accepted, this); > } catch (NotConnectedException e1) { > if(logMINOR) Logger.minor(this, "Lost connection to source"); > return; > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/2f615c5d/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 15:19:47 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 15:19:47 +0000 Subject: [freenet-dev] [freenet-cvs] r16833 - trunk/freenet/src/freenet/node In-Reply-To: <20071229014106.A62284796F8@freenetproject.org> References: <20071229014106.A62284796F8@freenetproject.org> Message-ID: <200801031519.48197.toad@amphibian.dyndns.org> On Saturday 29 December 2007 01:41, robert at freenetproject.org wrote: > Author: robert > Date: 2007-12-29 01:41:06 +0000 (Sat, 29 Dec 2007) > New Revision: 16833 > > Modified: > trunk/freenet/src/freenet/node/PeerNode.java > Log: > maybe help wont-fetch-ark deadlock > > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/PeerNode.java 2007-12-29 01:37:31 UTC (rev 16832) > +++ trunk/freenet/src/freenet/node/PeerNode.java 2007-12-29 01:41:06 UTC (rev 16833) > @@ -348,7 +348,7 @@ > disableRoutingHasBeenSetRemotely = false; // Assume so > > lastGoodVersion = fs.get("lastGoodVersion"); > - updateShouldDisconnectNow(); > + //updateShouldDisconnectNow(); > > testnetEnabled = fs.getBoolean("testnet", false); > if(node.testnetEnabled != testnetEnabled) { > @@ -2781,8 +2781,11 @@ > } > > synchronized void updateShouldDisconnectNow() { > - verifiedIncompatibleOlderVersion = forwardInvalidVersion(); > - verifiedIncompatibleNewerVersion = reverseInvalidVersion(); > + //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH THE NODE > + if (isConnected()) { > + verifiedIncompatibleOlderVersion = forwardInvalidVersion(); > + verifiedIncompatibleNewerVersion = reverseInvalidVersion(); > + } > } Please restore the original version. It looks like it was correct after all, and the problem is not fetching ARKs when verified*=true. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/cec91a45/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 15:23:14 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 15:23:14 +0000 Subject: [freenet-dev] [freenet-cvs] r16837 - trunk/freenet/src/freenet/node In-Reply-To: <20071229023617.0779F47B7A2@freenetproject.org> References: <20071229023617.0779F47B7A2@freenetproject.org> Message-ID: <200801031523.14611.toad@amphibian.dyndns.org> On Saturday 29 December 2007 02:36, robert at freenetproject.org wrote: > Author: robert > Date: 2007-12-29 02:36:16 +0000 (Sat, 29 Dec 2007) > New Revision: 16837 > > Modified: > trunk/freenet/src/freenet/node/FNPPacketMangler.java > Log: > opps, no... 'changeIP' is while connected, 'auth' is while disconnected (partially reverts r16836) It's possible in both cases afaics. So both commits should be reverted. No? > > Modified: trunk/freenet/src/freenet/node/FNPPacketMangler.java > =================================================================== > --- trunk/freenet/src/freenet/node/FNPPacketMangler.java 2007-12-29 02:12:19 UTC (rev 16836) > +++ trunk/freenet/src/freenet/node/FNPPacketMangler.java 2007-12-29 02:36:16 UTC (rev 16837) > @@ -237,7 +237,7 @@ > for(int i=0;i pn = peers[i]; > if(pn == opn) continue; > - if(pn.isConnected()) continue; > + if(!pn.isConnected()) continue; > if(tryProcess(buf, offset, length, pn.getCurrentKeyTracker(), now)) { > // IP address change > pn.changedIP(peer); > @@ -261,6 +261,7 @@ > for(int i=0;i pn = peers[i]; > if(pn == opn) continue; > + if(pn.isConnected()) continue; > if(tryProcessAuth(buf, offset, length, pn, peer,false, now)) return; > } > } > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/5f387ceb/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 15:25:55 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 15:25:55 +0000 Subject: [freenet-dev] [freenet-cvs] r16839 - trunk/freenet/src/freenet/node In-Reply-To: <20071229133433.A1D8447BBE8@freenetproject.org> References: <20071229133433.A1D8447BBE8@freenetproject.org> Message-ID: <200801031525.56456.toad@amphibian.dyndns.org> On Saturday 29 December 2007 13:34, nextgens at freenetproject.org wrote: > Author: nextgens > Date: 2007-12-29 13:34:33 +0000 (Sat, 29 Dec 2007) > New Revision: 16839 > > Modified: > trunk/freenet/src/freenet/node/FNPPacketMangler.java > Log: > jfk: Fix the issue spotted by robert (see http://archives.freenetproject.org/message/20071229.105331.b1a491f0.en.html) > > That might solve the "I've lost the exponential" problem It might? Wouldn't the old code cause an NPE? > > Modified: trunk/freenet/src/freenet/node/FNPPacketMangler.java > =================================================================== > --- trunk/freenet/src/freenet/node/FNPPacketMangler.java 2007-12-29 09:36:50 UTC (rev 16838) > +++ trunk/freenet/src/freenet/node/FNPPacketMangler.java 2007-12-29 13:34:33 UTC (rev 16839) > @@ -2640,8 +2640,8 @@ > return result; > } > } > - //FIXME: Isn't this wrong? 'result.myExponential' should be exponential? And what if dhContextToBePrunned is null? > - if((dhContextToBePrunned.myExponential).equals(result.myExponential)) > + > + if((dhContextToBePrunned != null) && ((dhContextToBePrunned.myExponential).equals(exponential))) > return dhContextToBePrunned; > } > return null; > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/5a926a5b/attachment.pgp From robert at emu.freenetproject.org Thu Jan 3 15:35:39 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Thu, 3 Jan 2008 09:35:39 -0600 Subject: [freenet-dev] [freenet-cvs] r16837 - trunk/freenet/src/freenet/node In-Reply-To: <200801031523.14611.toad@amphibian.dyndns.org> References: <20071229023617.0779F47B7A2@freenetproject.org> <200801031523.14611.toad@amphibian.dyndns.org> Message-ID: <29CC96D5-5368-43CF-8273-33BF6BEB0136@emu.freenetproject.org> On Jan 3, 2008, at 9:23 AM, Matthew Toseland wrote: > On Saturday 29 December 2007 02:36, robert at freenetproject.org wrote: >> Author: robert >> Date: 2007-12-29 02:36:16 +0000 (Sat, 29 Dec 2007) >> New Revision: 16837 >> >> Modified: >> trunk/freenet/src/freenet/node/FNPPacketMangler.java >> Log: >> opps, no... 'changeIP' is while connected, 'auth' is while >> disconnected > (partially reverts r16836) > > It's possible in both cases afaics. So both commits should be > reverted. No? Reverted in r16843 at nextgens request. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080103/e96fe279/attachment.htm From nextgens at freenetproject.org Thu Jan 3 15:56:47 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Thu, 3 Jan 2008 16:56:47 +0100 Subject: [freenet-dev] [freenet-cvs] r16839 - trunk/freenet/src/freenet/node In-Reply-To: <200801031525.56456.toad@amphibian.dyndns.org> References: <20071229133433.A1D8447BBE8@freenetproject.org> <200801031525.56456.toad@amphibian.dyndns.org> Message-ID: <20080103155647.GE4339@freenetproject.org> * Matthew Toseland [2008-01-03 15:25:55]: > On Saturday 29 December 2007 13:34, nextgens at freenetproject.org wrote: > > Author: nextgens > > Date: 2007-12-29 13:34:33 +0000 (Sat, 29 Dec 2007) > > New Revision: 16839 > > > > Modified: > > trunk/freenet/src/freenet/node/FNPPacketMangler.java > > Log: > > jfk: Fix the issue spotted by robert (see > http://archives.freenetproject.org/message/20071229.105331.b1a491f0.en.html) > > > > That might solve the "I've lost the exponential" problem > > It might? Wouldn't the old code cause an NPE? Well that's a fix-of-a-fix-of-a-fix, so yes, I won't dare saying it will fix it. No it wouldn't cause an npe... because of the way we instanciate the class... but it's a good practice to check for null there. If it was causing NPEs we would have spotted it earlier ;) NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/ff62597d/attachment.pgp From robert at emu.freenetproject.org Thu Jan 3 16:12:56 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Thu, 3 Jan 2008 10:12:56 -0600 Subject: [freenet-dev] [freenet-cvs] r16817 - in trunk/freenet/src/freenet: node support/math In-Reply-To: <200801031503.34926.toad@amphibian.dyndns.org> References: <20071226205138.AA33B47A371@freenetproject.org> <200801031503.34926.toad@amphibian.dyndns.org> Message-ID: On Jan 3, 2008, at 9:03 AM, Matthew Toseland wrote: > Useful class, but I'm really not convinced it's statistically sound. > You need > to normalize the stored value after every operation, otherwise as it > gets > larger, you will get bigger jumps when values are reported. In order > to do > this, you probably need to extend BDRA; put some hooks in. Hmmm... yes, in review you are right, as averaging is not linear as such: assume: avg(a, b, c... x, y) = z then: avg(a+1, b+1, c+1... x+1, y+1) != z+1 An implementation using normalized stored values is possible by adding a single package-hook: setCurrentValue(double d); Then, in the averager, on report() to call avg.setCurrentValue(Location.normalize(avg.currentValue()); Implemented in r16855. -- Robert Hailey > On Wednesday 26 December 2007 20:51, robert at freenetproject.org wrote: >> Author: robert >> Date: 2007-12-26 20:51:38 +0000 (Wed, 26 Dec 2007) >> New Revision: 16817 >> >> Added: >> trunk/freenet/src/freenet/support/math/DecayingKeyspaceAverage.java >> Modified: >> trunk/freenet/src/freenet/node/Location.java >> trunk/freenet/src/freenet/node/NodeStats.java >> Log: >> implement keyspace-aware averager -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080103/1067c949/attachment.htm From robert at emu.freenetproject.org Thu Jan 3 16:29:32 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Thu, 3 Jan 2008 10:29:32 -0600 Subject: [freenet-dev] [freenet-cvs] r16833 - trunk/freenet/src/freenet/node In-Reply-To: <200801031519.48197.toad@amphibian.dyndns.org> References: <20071229014106.A62284796F8@freenetproject.org> <200801031519.48197.toad@amphibian.dyndns.org> Message-ID: On Jan 2, 2008, at 7:38 AM, Florent Daigni?re wrote: >> Why not just use if(isConnected()) { ... } ? As far as I can see >> that is >> correct: if we're not connected, we don't care about verified*. > > You're suggesting to revert robert's patch, wich is fine by me... as I > said, I don't understand why it helps > > NextGen$ On Jan 3, 2008, at 9:19 AM, Matthew Toseland wrote: > Please restore the original version. It looks like it was correct > after all, > and the problem is not fetching ARKs when verified*=true. If you really want me to, I certainly will, but... if you and nextgens agree that it is benign to active connections, and in my reasoning prevents a deadlock for long disconnections... why? I think that it would be more useful to simply rename the identifiers to be more intuitive until the ark problem is solved to your satisfaction (e.g. as David said he may). -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080103/a3bc7665/attachment.htm From toad at amphibian.dyndns.org Thu Jan 3 16:30:36 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 16:30:36 +0000 Subject: [freenet-dev] [freenet-cvs] Verification of r16847 on emu In-Reply-To: <20080102165044.31E5C47BC01@freenetproject.org> References: <20080102165044.31E5C47BC01@freenetproject.org> Message-ID: <200801031630.42241.toad@amphibian.dyndns.org> Doh. Well it's technically a trivial refactoring; read the commit! On Wednesday 02 January 2008 16:50, svn-build at freenetproject.org wrote: > toad has declared that 16847 is an 'indent only' commit: > That's FALSE. > > Files orig/FileLoggerHook$WriterThread.class and new/FileLoggerHook$WriterThread.class differ > orig/FileLoggerHook$1.class : ec2113c9326737fecf85a3fde2029b6320496215 : ec2113c9326737fecf85a3fde2029b6320496215 > orig/FileLoggerHook.class : eefd01f8fc2283859e8abd8217d8a96eae2381d2 : eefd01f8fc2283859e8abd8217d8a96eae2381d2 > orig/FileLoggerHook$CloserThread.class : 0ea1a612efe4f3448e8b1d7385e33f2155298d34 : 0ea1a612efe4f3448e8b1d7385e33f2155298d34 > orig/FileLoggerHook$IntervalParseException.class : 5275ef6554ff1a86082cff5123c2cd2866737e41 : 5275ef6554ff1a86082cff5123c2cd2866737e41 > orig/FileLoggerHook$OldLogFile.class : 7cae6bc6c59831a86fd27596f411be688516a486 : 7cae6bc6c59831a86fd27596f411be688516a486 > orig/FileLoggerHook$WriterThread.class : dd82d2a08c1a0234400bccd04eaefbbfb0eaaa3e : 448c28f732b0a6804af921d7a4a0d3f05cc0b580 > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/43adbc14/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 16:49:45 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 16:49:45 +0000 Subject: [freenet-dev] [freenet-cvs] r16856 - trunk/freenet/src/freenet/clients/http In-Reply-To: <20080103161523.8FC5347B7A2@freenetproject.org> References: <20080103161523.8FC5347B7A2@freenetproject.org> Message-ID: <200801031649.50120.toad@amphibian.dyndns.org> On Thursday 03 January 2008 16:15, nextgens at freenetproject.org wrote: > Author: nextgens > Date: 2008-01-03 16:15:23 +0000 (Thu, 03 Jan 2008) > New Revision: 16856 > > Modified: > trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java > Log: > ConnectionToadlet: don't display the "add a peer" box for opennet > > Modified: trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java > =================================================================== > --- trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java 2008-01-03 16:11:15 UTC (rev 16855) > +++ trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java 2008-01-03 16:15:23 UTC (rev 16856) > @@ -466,9 +466,10 @@ > } > // END PEER TABLE > } > - > - drawAddPeerBox(contentNode, ctx); > > + if(!isOpennet()) > + drawAddPeerBox(contentNode, ctx); > + Override the callback. drawAddPeerBox should be inside if(shouldDrawNoderefBox()). > // our reference > if(shouldDrawNoderefBox(advancedModeEnabled)) > drawNoderefBox(contentNode, ctx); -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/8a4f8d8b/attachment.pgp From nextgens at freenetproject.org Thu Jan 3 16:49:57 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Thu, 3 Jan 2008 17:49:57 +0100 Subject: [freenet-dev] [freenet-cvs] Verification of r16847 on emu In-Reply-To: <200801031630.42241.toad@amphibian.dyndns.org> References: <20080102165044.31E5C47BC01@freenetproject.org> <200801031630.42241.toad@amphibian.dyndns.org> Message-ID: <20080103164956.GF4339@freenetproject.org> * Matthew Toseland [2008-01-03 16:30:36]: > Doh. Well it's technically a trivial refactoring; read the commit! > I'm not saying it isn't... bytecode wise it's different, that's all it tells :) NextGen$ > On Wednesday 02 January 2008 16:50, svn-build at freenetproject.org wrote: > > toad has declared that 16847 is an 'indent only' commit: > > That's FALSE. > > > > Files orig/FileLoggerHook$WriterThread.class and > new/FileLoggerHook$WriterThread.class differ > > orig/FileLoggerHook$1.class : ec2113c9326737fecf85a3fde2029b6320496215 : > ec2113c9326737fecf85a3fde2029b6320496215 > > orig/FileLoggerHook.class : eefd01f8fc2283859e8abd8217d8a96eae2381d2 : > eefd01f8fc2283859e8abd8217d8a96eae2381d2 > > orig/FileLoggerHook$CloserThread.class : > 0ea1a612efe4f3448e8b1d7385e33f2155298d34 : > 0ea1a612efe4f3448e8b1d7385e33f2155298d34 > > orig/FileLoggerHook$IntervalParseException.class : > 5275ef6554ff1a86082cff5123c2cd2866737e41 : > 5275ef6554ff1a86082cff5123c2cd2866737e41 > > orig/FileLoggerHook$OldLogFile.class : > 7cae6bc6c59831a86fd27596f411be688516a486 : > 7cae6bc6c59831a86fd27596f411be688516a486 > > orig/FileLoggerHook$WriterThread.class : > dd82d2a08c1a0234400bccd04eaefbbfb0eaaa3e : > 448c28f732b0a6804af921d7a4a0d3f05cc0b580 > > _______________________________________________ > > cvs mailing list > > cvs at freenetproject.org > > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > > > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/5650cb5b/attachment.pgp From nextgens at freenetproject.org Thu Jan 3 16:53:55 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Thu, 3 Jan 2008 17:53:55 +0100 Subject: [freenet-dev] [freenet-cvs] r16856 - trunk/freenet/src/freenet/clients/http In-Reply-To: <200801031649.50120.toad@amphibian.dyndns.org> References: <20080103161523.8FC5347B7A2@freenetproject.org> <200801031649.50120.toad@amphibian.dyndns.org> Message-ID: <20080103165355.GG4339@freenetproject.org> * Matthew Toseland [2008-01-03 16:49:45]: > On Thursday 03 January 2008 16:15, nextgens at freenetproject.org wrote: > > Author: nextgens > > Date: 2008-01-03 16:15:23 +0000 (Thu, 03 Jan 2008) > > New Revision: 16856 > > > > Modified: > > trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java > > Log: > > ConnectionToadlet: don't display the "add a peer" box for opennet > > > > Modified: trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java > > =================================================================== > > --- trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java > 2008-01-03 16:11:15 UTC (rev 16855) > > +++ trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java > 2008-01-03 16:15:23 UTC (rev 16856) > > @@ -466,9 +466,10 @@ > > } > > // END PEER TABLE > > } > > - > > - drawAddPeerBox(contentNode, ctx); > > > > + if(!isOpennet()) > > + drawAddPeerBox(contentNode, ctx); > > + > > Override the callback. drawAddPeerBox should be inside > if(shouldDrawNoderefBox()). done in r16858 NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/2035fbc1/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 17:24:11 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 17:24:11 +0000 Subject: [freenet-dev] [freenet-cvs] r16858 - trunk/freenet/src/freenet/clients/http In-Reply-To: <20080103165328.DDED347BC45@freenetproject.org> References: <20080103165328.DDED347BC45@freenetproject.org> Message-ID: <200801031724.11231.toad@amphibian.dyndns.org> I wouldn't even do that. Just have shouldDrawNoderefBox() return false, and have both drawAddPeerBox and drawNoderefBox conditional on it. That's cleanest. On Thursday 03 January 2008 16:53, you wrote: > Author: nextgens > Date: 2008-01-03 16:53:28 +0000 (Thu, 03 Jan 2008) > New Revision: 16858 > > Modified: > trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java > trunk/freenet/src/freenet/clients/http/OpennetConnectionsToadlet.java > Log: > ConnectionToadlet: same fix with different code > > Modified: trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java > =================================================================== > --- trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java 2008-01-03 16:23:39 UTC (rev 16857) > +++ trunk/freenet/src/freenet/clients/http/ConnectionsToadlet.java 2008-01-03 16:53:28 UTC (rev 16858) > @@ -467,8 +467,7 @@ > // END PEER TABLE > } > > - if(!isOpennet()) > - drawAddPeerBox(contentNode, ctx); > + drawAddPeerBox(contentNode, ctx); > > // our reference > if(shouldDrawNoderefBox(advancedModeEnabled)) > > Modified: trunk/freenet/src/freenet/clients/http/OpennetConnectionsToadlet.java > =================================================================== > --- trunk/freenet/src/freenet/clients/http/OpennetConnectionsToadlet.java 2008-01-03 16:23:39 UTC (rev 16857) > +++ trunk/freenet/src/freenet/clients/http/OpennetConnectionsToadlet.java 2008-01-03 16:53:28 UTC (rev 16858) > @@ -27,6 +27,10 @@ > PeerNodeStatus peerNodeStatus, boolean fProxyJavascriptEnabled) { > // Do nothing - no private notes either (no such thing as negative trust in cyberspace) > } > + > + protected void drawAddPeerBox(HTMLNode contentNode, ToadletContext ctx) { > + // Do nothing - we don't want opennet-refs dealing > + } > > protected boolean hasNameColumn() { > return false; > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/a615dff7/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 17:28:37 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 17:28:37 +0000 Subject: [freenet-dev] =?utf-8?q?=5Bfreenet-cvs=5D_r16839_-=09trunk/freene?= =?utf-8?q?t/src/freenet/node?= In-Reply-To: <20080103155647.GE4339@freenetproject.org> References: <20071229133433.A1D8447BBE8@freenetproject.org> <200801031525.56456.toad@amphibian.dyndns.org> <20080103155647.GE4339@freenetproject.org> Message-ID: <200801031728.38515.toad@amphibian.dyndns.org> On Thursday 03 January 2008 15:56, Florent Daigni?re wrote: > * Matthew Toseland [2008-01-03 15:25:55]: > > > On Saturday 29 December 2007 13:34, nextgens at freenetproject.org wrote: > > > Author: nextgens > > > Date: 2007-12-29 13:34:33 +0000 (Sat, 29 Dec 2007) > > > New Revision: 16839 > > > > > > Modified: > > > trunk/freenet/src/freenet/node/FNPPacketMangler.java > > > Log: > > > jfk: Fix the issue spotted by robert (see > > http://archives.freenetproject.org/message/20071229.105331.b1a491f0.en.html) > > > > > > That might solve the "I've lost the exponential" problem > > > > It might? Wouldn't the old code cause an NPE? > > Well that's a fix-of-a-fix-of-a-fix, so yes, I won't dare saying it > will fix it. > > No it wouldn't cause an npe... because of the way we instanciate the > class... but it's a good practice to check for null there. > > If it was causing NPEs we would have spotted it earlier ;) Which is why I'm not convinced the == null matters. But the comparing against the right thing does matter. > > NextGen$ > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/b12e2d68/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 17:53:00 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 17:53:00 +0000 Subject: [freenet-dev] [freenet-cvs] r16833 - trunk/freenet/src/freenet/node In-Reply-To: References: <20071229014106.A62284796F8@freenetproject.org> <200801031519.48197.toad@amphibian.dyndns.org> Message-ID: <200801031753.10300.toad@amphibian.dyndns.org> On Thursday 03 January 2008 16:29, Robert Hailey wrote: > > On Jan 2, 2008, at 7:38 AM, Florent Daigni?re wrote: > >> Why not just use if(isConnected()) { ... } ? As far as I can see > >> that is > >> correct: if we're not connected, we don't care about verified*. > > > > You're suggesting to revert robert's patch, wich is fine by me... as I > > said, I don't understand why it helps > > > > NextGen$ > > On Jan 3, 2008, at 9:19 AM, Matthew Toseland wrote: > > Please restore the original version. It looks like it was correct > > after all, > > and the problem is not fetching ARKs when verified*=true. > > If you really want me to, I certainly will, but... if you and nextgens > agree that it is benign to active connections, and in my reasoning > prevents a deadlock for long disconnections... why? I think that it > would be more useful to simply rename the identifiers to be more > intuitive until the ark problem is solved to your satisfaction (e.g. > as David said he may). No, IMHO we should unconditionally recompute verified*, as we did until recently. And not fetching ARKs if it's out of date is insane, which I've fixed in r16861. Please make it unconditionally recompute, and re-test to see if there is still a problem. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/be46246b/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 3 18:16:26 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 18:16:26 +0000 Subject: [freenet-dev] [freenet-cvs] r16817 - in trunk/freenet/src/freenet: node support/math In-Reply-To: References: <20071226205138.AA33B47A371@freenetproject.org> <200801031503.34926.toad@amphibian.dyndns.org> Message-ID: <200801031816.31390.toad@amphibian.dyndns.org> On Thursday 03 January 2008 16:12, Robert Hailey wrote: > > On Jan 3, 2008, at 9:03 AM, Matthew Toseland wrote: > > > Useful class, but I'm really not convinced it's statistically sound. > > You need > > to normalize the stored value after every operation, otherwise as it > > gets > > larger, you will get bigger jumps when values are reported. In order > > to do > > this, you probably need to extend BDRA; put some hooks in. > > Hmmm... yes, in review you are right, as averaging is not linear as > such: > assume: avg(a, b, c... x, y) = z > then: avg(a+1, b+1, c+1... x+1, y+1) != z+1 > > An implementation using normalized stored values is possible by adding > a single package-hook: setCurrentValue(double d); > Then, in the averager, on report() to call > avg.setCurrentValue(Location.normalize(avg.currentValue()); > > Implemented in r16855. Cool. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/b3004443/attachment.pgp From robert at emu.freenetproject.org Thu Jan 3 18:19:48 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Thu, 3 Jan 2008 12:19:48 -0600 Subject: [freenet-dev] [freenet-cvs] r16833 - trunk/freenet/src/freenet/node In-Reply-To: <200801031753.10300.toad@amphibian.dyndns.org> References: <20071229014106.A62284796F8@freenetproject.org> <200801031519.48197.toad@amphibian.dyndns.org> <200801031753.10300.toad@amphibian.dyndns.org> Message-ID: <830B6A19-DA08-4BE2-95FB-805973C0A0B5@emu.freenetproject.org> On Jan 3, 2008, at 11:53 AM, Matthew Toseland wrote: > On Thursday 03 January 2008 16:29, Robert Hailey wrote: >> >> On Jan 3, 2008, at 9:19 AM, Matthew Toseland wrote: >>> Please restore the original version. It looks like it was correct >>> after all, >>> and the problem is not fetching ARKs when verified*=true. >> >> If you really want me to, I certainly will, but... if you and >> nextgens >> agree that it is benign to active connections, and in my reasoning >> prevents a deadlock for long disconnections... why? I think that it >> would be more useful to simply rename the identifiers to be more >> intuitive until the ark problem is solved to your satisfaction (e.g. >> as David said he may). > > No, IMHO we should unconditionally recompute verified*, as we did > until > recently. And not fetching ARKs if it's out of date is insane, which > I've > fixed in r16861. Please make it unconditionally recompute, and re- > test to see > if there is still a problem. Restored in r16862, any objection to new function/variable names to suite the newer purpose? updateShouldDisconnectNow -> updateVersionRoutablity verifiedIncompatibleOlderVersion -> unroutableOlderVersion verifiedIncompatibleNewerVersion -> unroutableNewerVersion isVerifiedIncompatibleOlderVersion -> isUnroutableOlderVersion isVerifiedIncompatibleNewerVersion -> isUnroutableNewerVersion And remove/change old comments: e.g. "on a relevant incoming handshake" (ln#85), "breaking the meaning of verifiedIncompable[Older| Newer]Version" (ln#2794)? -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080103/72e90aab/attachment.htm From freenet-devl at david.sowder.com Thu Jan 3 19:03:47 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Thu, 03 Jan 2008 13:03:47 -0600 Subject: [freenet-dev] [freenet-cvs] r16833 - trunk/freenet/src/freenet/node In-Reply-To: <830B6A19-DA08-4BE2-95FB-805973C0A0B5@emu.freenetproject.org> References: <20071229014106.A62284796F8@freenetproject.org> <200801031519.48197.toad@amphibian.dyndns.org> <200801031753.10300.toad@amphibian.dyndns.org> <830B6A19-DA08-4BE2-95FB-805973C0A0B5@emu.freenetproject.org> Message-ID: <477D3193.9010904@david.sowder.com> I'd have to extensively review the current PeerNode code to make sure my understanding isn't outdated, but I prefer the current variable names which contain the idea of their value being verified and aren't mixed up directly with the idea of routability. OTOH, I'm probably a little biased, they being originally my variable names IIRC. Robert Hailey wrote: > > On Jan 3, 2008, at 11:53 AM, Matthew Toseland wrote: >> On Thursday 03 January 2008 16:29, Robert Hailey wrote: >>> On Jan 3, 2008, at 9:19 AM, Matthew Toseland wrote: >>>> Please restore the original version. It looks like it was correct >>>> after all, >>>> and the problem is not fetching ARKs when verified*=true. >>> >>> If you really want me to, I certainly will, but... if you and nextgens >>> agree that it is benign to active connections, and in my reasoning >>> prevents a deadlock for long disconnections... why? I think that it >>> would be more useful to simply rename the identifiers to be more >>> intuitive until the ark problem is solved to your satisfaction (e.g. >>> as David said he may). >> >> No, IMHO we should unconditionally recompute verified*, as we did until >> recently. And not fetching ARKs if it's out of date is insane, which >> I've >> fixed in r16861. Please make it unconditionally recompute, and >> re-test to see >> if there is still a problem. > > Restored in r16862, any objection to new function/variable names to > suite the newer purpose? > > updateShouldDisconnectNow -> updateVersionRoutablity > verifiedIncompatibleOlderVersion -> unroutableOlderVersion > verifiedIncompatibleNewerVersion -> unroutableNewerVersion > isVerifiedIncompatibleOlderVersion -> isUnroutableOlderVersion > isVerifiedIncompatibleNewerVersion -> isUnroutableNewerVersion > > And remove/change old comments: e.g. "on a relevant incoming > handshake" (ln#85), "breaking the meaning of > verifiedIncompable[Older|Newer]Version" (ln#2794)? From toad at amphibian.dyndns.org Thu Jan 3 23:51:33 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 3 Jan 2008 23:51:33 +0000 Subject: [freenet-dev] Freenet 0.7 build 1097 Message-ID: <200801032351.38220.toad@amphibian.dyndns.org> Freenet 0.7 build 1097 is now available. Please upgrade. This is mostly bugfixes, but some of them are fairly significant. There is much improved support for running the node on GCJ thanks to xor, although much remains to be done (the web interface isn't very responsive on GCJ for example). There are bugfixes to connection setup, ARK fetching, requests, at least one deadlock, statistics collection, and more. If you find bugs, please report them via the bug tracker at https://bugs.freenetproject.org/ or via Frost (bearing in mind that several boards are under attack at the moment), the mailing lists, or IRC. Thanks. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080103/c0bd41f5/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:00:49 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:00:49 +0000 Subject: [freenet-dev] [freenet-cvs] r16833 - trunk/freenet/src/freenet/node In-Reply-To: <830B6A19-DA08-4BE2-95FB-805973C0A0B5@emu.freenetproject.org> References: <20071229014106.A62284796F8@freenetproject.org> <200801031753.10300.toad@amphibian.dyndns.org> <830B6A19-DA08-4BE2-95FB-805973C0A0B5@emu.freenetproject.org> Message-ID: <200801040000.50428.toad@amphibian.dyndns.org> On Thursday 03 January 2008 18:19, Robert Hailey wrote: > > On Jan 3, 2008, at 11:53 AM, Matthew Toseland wrote: > > On Thursday 03 January 2008 16:29, Robert Hailey wrote: > >> > >> On Jan 3, 2008, at 9:19 AM, Matthew Toseland wrote: > >>> Please restore the original version. It looks like it was correct > >>> after all, > >>> and the problem is not fetching ARKs when verified*=true. > >> > >> If you really want me to, I certainly will, but... if you and > >> nextgens > >> agree that it is benign to active connections, and in my reasoning > >> prevents a deadlock for long disconnections... why? I think that it > >> would be more useful to simply rename the identifiers to be more > >> intuitive until the ark problem is solved to your satisfaction (e.g. > >> as David said he may). > > > > No, IMHO we should unconditionally recompute verified*, as we did > > until > > recently. And not fetching ARKs if it's out of date is insane, which > > I've > > fixed in r16861. Please make it unconditionally recompute, and re- > > test to see > > if there is still a problem. > > Restored in r16862, any objection to new function/variable names to > suite the newer purpose? > > updateShouldDisconnectNow -> updateVersionRoutablity > verifiedIncompatibleOlderVersion -> unroutableOlderVersion > verifiedIncompatibleNewerVersion -> unroutableNewerVersion > isVerifiedIncompatibleOlderVersion -> isUnroutableOlderVersion > isVerifiedIncompatibleNewerVersion -> isUnroutableNewerVersion > > And remove/change old comments: e.g. "on a relevant incoming > handshake" (ln#85), "breaking the meaning of verifiedIncompable[Older| > Newer]Version" (ln#2794)? No objections here, it's a good idea. Commit the comments separately. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/a56fa115/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:01:08 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:01:08 +0000 Subject: [freenet-dev] =?iso-8859-1?q?=5Bfreenet-cvs=5D_r16833_-=09trunk/f?= =?iso-8859-1?q?reenet/src/freenet/node?= In-Reply-To: <477D3193.9010904@david.sowder.com> References: <20071229014106.A62284796F8@freenetproject.org> <830B6A19-DA08-4BE2-95FB-805973C0A0B5@emu.freenetproject.org> <477D3193.9010904@david.sowder.com> Message-ID: <200801040001.09145.toad@amphibian.dyndns.org> On Thursday 03 January 2008 19:03, David Sowder wrote: > I'd have to extensively review the current PeerNode code to make sure my > understanding isn't outdated, but I prefer the current variable names > which contain the idea of their value being verified and aren't mixed up > directly with the idea of routability. OTOH, I'm probably a little > biased, they being originally my variable names IIRC. The way they are being used now, verified doesn't make any sense. > > Robert Hailey wrote: > > > > On Jan 3, 2008, at 11:53 AM, Matthew Toseland wrote: > >> On Thursday 03 January 2008 16:29, Robert Hailey wrote: > >>> On Jan 3, 2008, at 9:19 AM, Matthew Toseland wrote: > >>>> Please restore the original version. It looks like it was correct > >>>> after all, > >>>> and the problem is not fetching ARKs when verified*=true. > >>> > >>> If you really want me to, I certainly will, but... if you and nextgens > >>> agree that it is benign to active connections, and in my reasoning > >>> prevents a deadlock for long disconnections... why? I think that it > >>> would be more useful to simply rename the identifiers to be more > >>> intuitive until the ark problem is solved to your satisfaction (e.g. > >>> as David said he may). > >> > >> No, IMHO we should unconditionally recompute verified*, as we did until > >> recently. And not fetching ARKs if it's out of date is insane, which > >> I've > >> fixed in r16861. Please make it unconditionally recompute, and > >> re-test to see > >> if there is still a problem. > > > > Restored in r16862, any objection to new function/variable names to > > suite the newer purpose? > > > > updateShouldDisconnectNow -> updateVersionRoutablity > > verifiedIncompatibleOlderVersion -> unroutableOlderVersion > > verifiedIncompatibleNewerVersion -> unroutableNewerVersion > > isVerifiedIncompatibleOlderVersion -> isUnroutableOlderVersion > > isVerifiedIncompatibleNewerVersion -> isUnroutableNewerVersion > > > > And remove/change old comments: e.g. "on a relevant incoming > > handshake" (ln#85), "breaking the meaning of > > verifiedIncompable[Older|Newer]Version" (ln#2794)? > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/54985136/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:03:31 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:03:31 +0000 Subject: [freenet-dev] [freenet-cvs] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <477C303C.3040504@david.sowder.com> References: <20071229014631.C5D7847A6AE@freenetproject.org> <200801022140.35033.toad@amphibian.dyndns.org> <477C303C.3040504@david.sowder.com> Message-ID: <200801040003.32684.toad@amphibian.dyndns.org> I disagree. The way they are used now, they just indicate whether the node is routable, and for what reason. The only reason for them being like that in the past was that we would connect to nodes which are unroutable, and then disconnect and periodically do version probes. We don't need to do this now, because we keep open connections to incompatible nodes for purposes of Update Over Mandatory. On Thursday 03 January 2008 00:45, David Sowder (Zothar) wrote: > In my thinking, we should only change verifiedIncompatibleOlderVersion > and verifiedIncompatibleNewerVersion in four situations, all of which > may even mean that we call a function/method to check for the need to > change in the code paths that are common to each of these situations: > > 1) Potentially set when we've just completed a handshake and therefore > are connected and isConnected() is true (not necessarily isRoutable() > for whatever reason) after which we have no need to fetch an ARK for the > peer > 2) Cleared if we are disconnecting (for whatever reason) from a peer > that has had either verifiedIncompatibleOlderVersion or > verifiedIncompatibleNewerVersion set, after which we may want to fetch > an ARK for the peer just like we would if we had never connected to them > since our last restart > 3) Set verifiedIncompatibleOlderVersion when our node passes a mandatory > threshold date and time > 4) Set verifiedIncompatibleNewerVersion when the peer tells us we are > too old for their idea of the mandatory threshold date and time (I don't > recall by what mechanism this is accomplished ATM) > > Once verifiedIncompatibleOlderVersion or > verifiedIncompatibleNewerVersion is set for a peer, it cannot be cleared > by any means other than the peer reconnecting (either by a intermediate > DISCONNECTED, etc. state or by boot ID change (if I understand that > name/concept correctly)) after their restart or by our node's > initializing the value to it's default False value on our node's restart. > > Caveat the above by the fact that I haven't read this related code in > awhile, so I'm going by my memory of how it should work and what I > remember of changes made by others since. > > Matthew Toseland wrote: > > On Wednesday 02 January 2008 18:30, David Sowder wrote: > > > >> I'm not sure of the thinking of others regarding changes to the > >> verifiedIncompatible variables since, but my thinking in my initial > >> implementation was that they would only be set after a handshake, thus > >> we already had a connected to them and ARKs weren't needed. That won't > >> change until the peer re-handshakes after restarting with a different > >> version of the node's software. > >> > > > > I don't follow - clearly if isConnected(), we don't need the ARK. > > > > ------------------------------------------------------------------------ > > > > _______________________________________________ > > Devl mailing list > > Devl at freenetproject.org > > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/a80e2a75/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:05:20 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:05:20 +0000 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <477C315D.8010609@david.sowder.com> References: <20071229014631.C5D7847A6AE@freenetproject.org> <010C7EF2-50D3-4E08-AC05-7F1F4E281FB0@emu.freenetproject.org> <477C315D.8010609@david.sowder.com> Message-ID: <200801040005.20664.toad@amphibian.dyndns.org> On Thursday 03 January 2008 00:50, David Sowder (Zothar) wrote: > To be clear, a TOO OLD/TOO NEW peer is currently connected in the sense > that the node is exchanging traffic with it. The node just doesn't > route any traffic through the peer and the traffic with the peer is > limited to keeping the connectivity, N2NMs and UoM exchanges (I don't > believe I'm forgetting anything, but it wouldn't be the first time). Yes. > > Robert Hailey wrote: > > On Jan 2, 2008, at 3:44 PM, Matthew Toseland wrote: > > > > > >> On Wednesday 02 January 2008 17:22, Robert Hailey wrote: > >> > >>> On Dec 29, 2007, at 5:39 AM, Florent Daigni?re wrote: > >>> > >>> > >>>>>> synchronized void updateShouldDisconnectNow() { > >>>>>> //FIXME: We should not update VERIFIED unless we HANDSHAKE WITH > >>>>>> THE NODE > >>>>>> - if (isConnected()) { > >>>>>> + if (isConnected() || verifiedIncompatibleOlderVersion || > >>>>>> verifiedIncompatibleNewerVersion) { > >>>>>> verifiedIncompatibleOlderVersion = forwardInvalidVersion(); > >>>>>> verifiedIncompatibleNewerVersion = reverseInvalidVersion(); > >>>>>> } > >>>>>> > >>>>> I suggest you call isRoutable() instead > >>>>> > >>>>> NextGen$ > >>>>> > >>>> Hmm, on a latter thought, I don't understand why it helps... nor why > >>>> we > >>>> are calling it from PacketSender > >>>> > >>>> What about getting rid of both the "if" branch and the call in > >>>> PacketSender ? > >>>> > >>> In PacketSender this function is called ~precisely~ at the version > >>> transition time so that all peers will be re-evaluated concerning > >>> compatible versioning; this seems like an odd place to stick this > >>> code, but also updateShouldDisconnectNow() does not explicitly > >>> disconnect the peer node, it only updates the 'incompatible' flags. > >>> > >> We don't want to disconnect. We want to stay connected and to not > >> route > >> requests to the node or accept them from it. We will still accept > >> e.g. Update > >> Over Mandatory traffic. > >> > > > > Fair enough, but this is only confused semantics. I said disconnect as > > it is in the function name, perhaps a better name would be > > updateVersionRoutablity(). > > > > > >>> Concerning the if branch I added, the intended behaviour change is > >>> that this function should only (1) allow the disconnection of a > >>> connected peer by setting the incompatible flag, > >>> > >> No, we don't want to disconnect. We just want to make it non- > >> routable, by > >> updating verified*. > >> > > > > I used the wrong term. It will only update the verified* booleans (and > > thus make routable or not) if connected. But see below... > > > > > >>> Now, if the nodes can handshake, this code has probably has little > >>> effect, as it simply further restricts one of only two places that > >>> the > >>> incompatible flags are set (the other being on a handshake). What it > >>> will do is if the nodes can't talk to each other (like in my re-keyed > >>> node case), is not show 'TOO_OLD'/'TOO_NEW', but instead > >>> 'DISCONNECTED'. > >>> > >> So you are setting verified* to false, because it's not verified? > >> > > > > Not setting it to false, but... not allowing it to be set to true if > > that node cannot reach us (e.g. disconnected). > > > > > >>> And I suppose in the same error case... if the node is > >>> restarted, it would never be able to verify that the peer is too old/ > >>> new and fetch the ark (keeping the version number up-to-date). > >>> > >> Again, we should always fetch the ARK. > >> > > > > The original implementation appears to use verifiedIncompatible* to > > optimize against a time-based trigger (Version-Incompatibility) from a > > bunch of ark requests being started. > > > > On Jan 2, 2008, at 12:30 PM, David Sowder wrote: > > > >> I'm not sure of the thinking of others regarding changes to the > >> verifiedIncompatible variables since, but my thinking in my initial > >> implementation was that they would only be set after a handshake, thus > >> we already had a connected to them and ARKs weren't needed. That > >> won't > >> change until the peer re-handshakes after restarting with a different > >> version of the node's software. > >> > > > > As best I can see, the verified* booleans have changed meanings since > > update-over-mandatory. If it is still useful to know if the peer node > > is presently-verified-{older/newer}, then we should split the booleans > > (isOlder,isVerifiedOlder); otherwise just fetch an ark even if it is > > an older (as Matthew suggested). In either case the misleading > > comments and variable names should be removed and/or changed. > > > > -- > > Robert Hailey > > > > _______________________________________________ > > Devl mailing list > > Devl at freenetproject.org > > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/728ddf6e/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:07:02 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:07:02 +0000 Subject: [freenet-dev] r16834 - trunk/freenet/src/freenet/node In-Reply-To: <477CF21C.7000301@david.sowder.com> References: <20071229014631.C5D7847A6AE@freenetproject.org> <20080103135723.GD4339@freenetproject.org> <477CF21C.7000301@david.sowder.com> Message-ID: <200801040007.02737.toad@amphibian.dyndns.org> I agree with Zothar. On Thursday 03 January 2008 14:33, David Sowder wrote: > >>>>>> > >>>>> We want to fetch ARKs in any case imo. > >>>>> > >>>>> > >>>> And in my opinion, fetching an ARK for a connected peer is pointless, but > >>>> perhaps we agree in our assessments and I misunderstand by thinking you > >>>> mean to _always_ fetch ARKs regardless of current peer status. > >>>> > >>>> > >>> It's not pointless. We want to keep an up to date "edition" for the ark so > >>> that when we will need it we will be able to reach it... Of course we > >>> could exchange and update the edition number on handshake... but atm we > >>> don't. > >>> > >>> > >> It is pointless. If the problem is that we don't have updated ARK edition > >> numbers, then we should fix that by exchanging that across an existing > >> connection when we have one rather than forcing the use of a backwards and > >> network load causing method for for information we have from a directly > >> connected peer. > >> > > > > It has at least two assets : 1) code simplicity 2) it propagates the ARK > > > IMO, both 1 and 2 are outweighed by the network load of all nodes > constantly fetching ARKs for all of their peers network-wide. > > 1) IMO, this is a flimsy argument considering that there are many parts > of the Freenet code base that are much more complex than what I'm > talking about would require and many of them, while perhaps they could > be simpler, have to be somewhat complex by necessity. > 2) Fetching the ARK on first connect and some number of minutes or hours > after the peer tells us via differential references that it has changed > would have a much more gentle impact on network load. It could be > argued that if ARKs aren't propagating from their inserts correctly, > there is a routing issue. If we're worried about the ARK falling out of > the network, we could fetch it once every seven days after it's last change. > >> Fixing that is simply a matter of exchanging either full > >> or differential references with our peers any time our node's reference > >> changes. > >> > > > > Lots of things are that simple and in need of doing because noone is > > available to implement them. > > > Perhaps this one interests me enough to code up if I'm in the right mood > after work some day in the coming weeks. I'll probably implement it on > top of N2NMs since they were designed to be generic and I see no reason > not to use them as they already support sending arbitrary SFS blocks. > >> I'm not exactly available to implement it right now, but I know > >> for a fact that this is not a hard problem to solve. > >> > > > > Neither am I > > > > > >> In fact, differential > >> references could be trivially implemented using the existing N2NM messaging > >> facilities. > >> > > > > Creating a new DMT message type or extending an existing one is the way > > to go. > > > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/1743c8eb/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:14:40 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:14:40 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <477C379F.4050706@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801022352.54143.toad@amphibian.dyndns.org> <477C379F.4050706@cs.ucl.ac.uk> Message-ID: <200801040014.40597.toad@amphibian.dyndns.org> On Thursday 03 January 2008 01:17, Michael Rogers wrote: > Matthew Toseland wrote: > > - It should be possible with such a simple request/response to make it > > extremely difficult for two nodes to realise they are on the same path, > > except in the case where they are actually adjacent. > > I think that's optimistic - they could use the tunnel construction time, > or the delay between the tunnel being constructed and used, or (for > two-way tunnels) the delay between the request and the response. The tunnel is constructed onion-style during the request's forwarding. The return tunnel likewise. There is no tunnel construction time. > > Let's say the attacker controls the exit node of tunnel X, and it > normally takes 5 seconds to construct a tunnel. If an interesting > request is sent through tunnel X, all the attacker's nodes record > predecessor samples for any outgoing tunnels constructed in the 5 > seconds before tunnel X was constructed, and successor samples for any > incoming tunnels constructed in the 5 seconds after tunnel X was > constructed. It doesn't matter that most of the samples are wrong - all > that matters is that the initiator is more likely to be recorded than a > random node (or a random member of the cell). Ok, this is possible, although slow. Is it fatal? How much data does the attacker need? > > If the attacker knows that tunnels always contain more than two nodes > and don't pass through any node more than once, he can also eliminate > samples from the exit node and its immediate neighbours. But that's not > vital, it just reduces the level of noise. > > > However, it *is* > > known that it is one of the nodes along that path that is the problem, and > > this may be enough. > > Yup, if we can trust the topological information then we should be able > to identify unreliable nodes and avoid using them in tunnels (but on the > other hand that allows an attacker to attract more tunnels and thus > gather more samples by performing well, as in I2P). > > > Each node publishes its links, including its pubkeys, their pubkey hashes, and > > a signature. This is relayed to some distance. We compute a credibility value > > for each node on the graph by starting with a value of 1.0 on each node we > > are a direct peer of, and dividing it by the number of peers on each node > > going outwards, adding up the values from different branches. We can then > > ignore nodes which have a low credibility value. > > Sounds promising, but the result will be different from each node's > point of view. How do we ensure that all members of the cell agree about > which nodes to ignore? If they don't agree then it's easy for an > attacker to calculate credibility values from every other node's point > of view in order to work out who would or wouldn't have chosen a given > exit node. I don't know. IMHO we need to have a fixed cell which every node within it uses, in order to avoid a range of attacks. > > > Now, we may not be able to > > do exactly this, if we have to have semi-permanent cell structures, but it > > should be possible to find an algorithm to serve this purpose (eigenvector > > centrality? TrustRank etc?). > > We're caught in a cleft stick here: algorithms that give a universally > agreed score for each node are vulnerable to Sybil attacks [1], but if a > node's score depends on who evaluates it (eg the flow-based algorithm > you sketched above) then how do we agree about which nodes to ignore? Well can we just use it from one node's perspective? When I last looked into it that looked very risky. > > Cheers, > Michael > > [1] http://www.sigcomm.org/sigcomm2005/paper-CheFri.pdf -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/5682dbe6/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:16:28 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:16:28 +0000 Subject: [freenet-dev] Why getting rid of HTL is bad was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <477C3BEF.9090308@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801030000.13355.toad@amphibian.dyndns.org> <477C3BEF.9090308@cs.ucl.ac.uk> Message-ID: <200801040016.28579.toad@amphibian.dyndns.org> On Thursday 03 January 2008 01:35, Michael Rogers wrote: > Matthew Toseland wrote: > > No, but it would make the request time variance much higher, especially for > > inserts and unsuccessful requests (which can potentially visit the entire > > network). > > True, the variance would be higher. Visiting the entire network is a bit > of an extreme example though - in theory it's also possible with the > current hop counter, but it's not exactly likely. :-) > > > A lot more requests will naturally timeout than do now, and therefore a node > > can silently drop a request without arousing much concern or suspicion. Per > > node failure tables can't reroute that key until the timeout happens, and if > > the timeout is larger than it is now, that takes longer. > > We're using a reliable transport between nodes, so the only things that > should cause a timeout are overloaded, crashed or faulty nodes. Not true. Very long paths can also cause this. > If we > can keep the ping time under control then we can detect crashed nodes > quickly. So with proper load management we should be able to pretty much > eliminate timeouts except in the case of faulty nodes, which can then be > detected. Not with probabilistic termination. > > >> I shouldn't have said we can't analyse it, but I wouldn't know where to > >> start because so much depends on the topology. For example, the > >> closest-location-so-far obviously leaks information, but how do we > >> quantify it? > > > > Isn't it similar to the situation with keys? > > Sorry, I don't follow. > > The closest-location-so-far tells an attacker something about the path a > request has followed - among other things, it allows the attacker to > rule out certain nodes the request definitely hasn't passed through. But > how would we quantify that in terms of the size of the initiator's > anonymity set or the probability that a given node is the initiator, for > example? Well, how does the attacker rule out previous nodes? How many can he rule out? > > Whereas with a weighted coin, if the probability that each node > terminates the request is 1/n then there's a 1/n chance that the > previous hop is the initiator. > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/55db83f8/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:20:53 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:20:53 +0000 Subject: [freenet-dev] Tunnels vs premix routing was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <477C3F20.7070208@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801030008.57919.toad@amphibian.dyndns.org> <477C3F20.7070208@cs.ucl.ac.uk> Message-ID: <200801040020.53673.toad@amphibian.dyndns.org> On Thursday 03 January 2008 01:49, Michael Rogers wrote: > Matthew Toseland wrote: > > "Local" = direct peer. If we use one tunnel per local pseudo-identity, it > > might work, but there is still a very high chance that the node sending the > > request is the originator. > > Right, but that's also the case at the moment: if requests travel n hops > on average then there's a 1/n chance that the previous hop is the > initiator. I don't see how you can get away from that without onion routing. Indeed. Which is why afaics we need onion routing. Would tunneling without onion routing increase our vulnerability? It looks like it might to me. > > > OTOH if we use more than one tunnel, either local > > collusion or rerouting may make life *much* easier for the attacker. > > Yes, we should keep the number of linkable tunnels to a minimum. > > > Makes it easier to track down the origin node for a swarm of requests once it > > reaches the request stage. Maybe that's not such a big deal. > > If tunnels do their job then it doesn't matter if an attacker can find > the last node in the tunnel. It's still good to avoid it if possible. Consider it a minor issue. > > > For performance most users would likely want the node to use multiple tunnels > > for a single splitfile, and arguably this is a security issue: if the > > attacker receives the tunnel, and doesn't like its contents, he can > > trickle-feed it as an effective DoS. > > Users who value performance over anonymity could use multiple tunnels > per splitfile and/or shorter tunnels. I'm just saying that for maximum > anonymity, the tunnels should be long enough that any node could be the > initiator, and the number of linkable tunnels should be minimised (eg > one tunnel per Frost ID per session rather than one per message). Sure, and from time to time they will be DoSed and they will have to reroute. Or they will route through a series of overloaded nodes and have to reroute. Even here we end up with multiple tunnels. > > > For > > premix routing, any effective attack requires that the tunnel exit node is > > owned by the attacker, and usually at least one node prior to that also. > > True, I'm not suggesting that tunnels are as strong as premix routing, > but on the other hand they don't require revealing the topology or > distributing (and agreeing on) public keys for every node in the cell. Yes but swapping already requires we reveal a lot of the topology, that's not a big deal IMHO. It partly depends on what you think about local attackers - without premix routing, local attackers are extremely powerful. > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/9f5d4692/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:35:20 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:35:20 +0000 Subject: [freenet-dev] Predecessor sampling and tunnel padding was Re: One tunnel per request: justification In-Reply-To: <477C379F.4050706@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801022352.54143.toad@amphibian.dyndns.org> <477C379F.4050706@cs.ucl.ac.uk> Message-ID: <200801040035.26492.toad@amphibian.dyndns.org> On Thursday 03 January 2008 01:17, Michael Rogers wrote: > Matthew Toseland wrote: > > - It should be possible with such a simple request/response to make it > > extremely difficult for two nodes to realise they are on the same path, > > except in the case where they are actually adjacent. > > I think that's optimistic - they could use the tunnel construction time, > or the delay between the tunnel being constructed and used, or (for > two-way tunnels) the delay between the request and the response. > > Let's say the attacker controls the exit node of tunnel X, and it > normally takes 5 seconds to construct a tunnel. If an interesting > request is sent through tunnel X, all the attacker's nodes record > predecessor samples for any outgoing tunnels constructed in the 5 > seconds before tunnel X was constructed, and successor samples for any > incoming tunnels constructed in the 5 seconds after tunnel X was > constructed. It doesn't matter that most of the samples are wrong - all > that matters is that the initiator is more likely to be recorded than a > random node (or a random member of the cell). Okay, so it's a predecessor sample, a very noisy source, but it will eventually reveal the originator if one of the attacker's nodes is connected to it. And the frequency data will also eventually show nodes close to the originator but not equal to it. So suppose we use long lived tunnels. Tagging on long-lived tunnels is trivial. So we have a high quality sample on every new tunnel. It will be essential to have multiple tunnels for adequate performance and reliability. What is the impact of the node creating two parallel tunnels each through one of its peer nodes which is run by the attacker? If he can link them, there is still the possibility that they are both from intermediary nodes from another originator - but this is probably less likely than the node being the originator, so if we have any long term data we can catch him. What about tunnel padding? I know a lot of people think this is impossible, and it would be a performance issue, but here's a proposal I made a while ago; as far as I can see, it would make tagging impossible except when we have on the chain attacker - relay - attacker. ----------------------------------------- For each tunnel, we send exactly one 512 byte packet every second. This has a 16-byte MAC internally, and externally has a flag. The former is encrypted, and generated by the endpoint; the latter is readable as plaintext by the node we forward the packet to. There may also be an external MAC, and an internal sequence number, but these are irrelevancies. If the internal MAC is invalid on a packet, the endpoint silently drops the packet. The external flag is true if the packet was forwarded (usually), and false if it is bogus. If in a second we need to send a packet down the tunnel, and we don't have a packet to send, we send a bogus packet. We generate a random 512 byte packet, including the MAC, and send this to the next hop, with the flag turned on. If in a second we have too many queued packets to send, we drop one. If we know some are bogus, we preferentially drop them. Now, when we forward a bogus packet, we make it un-bogus. All we have to do is turn the flag back off. Thus, the next node on the chain can identify that the packet was bogus, but this is irrelevant because he knows he is on the same tunnel anyway, if both nodes are run by an attacker (duh!). If the two nodes are not in direct line on the tunnel, then it is very difficult for them to establish that they are on the same tunnel (admittedly there are setup timing issues and so on; these can be solved by having tunnel setup take a long time). This wipes out a huge swathe of attacks. Admittedly it does so at a considerable cost in bandwidth! ---------------------------------------------------- > > If the attacker knows that tunnels always contain more than two nodes > and don't pass through any node more than once, he can also eliminate > samples from the exit node and its immediate neighbours. But that's not > vital, it just reduces the level of noise. > > > However, it *is* > > known that it is one of the nodes along that path that is the problem, and > > this may be enough. > > Yup, if we can trust the topological information then we should be able > to identify unreliable nodes and avoid using them in tunnels (but on the > other hand that allows an attacker to attract more tunnels and thus > gather more samples by performing well, as in I2P). -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/8eea0eb4/attachment.pgp From m.rogers at cs.ucl.ac.uk Fri Jan 4 00:37:18 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 00:37:18 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <200801040014.40597.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801022352.54143.toad@amphibian.dyndns.org> <477C379F.4050706@cs.ucl.ac.uk> <200801040014.40597.toad@amphibian.dyndns.org> Message-ID: <477D7FBE.9020506@cs.ucl.ac.uk> Matthew Toseland wrote: > The tunnel is constructed onion-style during the request's forwarding. The > return tunnel likewise. There is no tunnel construction time. Ah cool, I see what you mean now. >> It doesn't matter that most of the samples are wrong - all >> that matters is that the initiator is more likely to be recorded than a >> random node (or a random member of the cell). > > Ok, this is possible, although slow. Is it fatal? How much data does the > attacker need? Hard to say in general - it depends on an awful lot of variables. But we know that increasing the number of linkable tunnels increases the number of samples the attacker can collect. >> We're caught in a cleft stick here: algorithms that give a universally >> agreed score for each node are vulnerable to Sybil attacks [1], but if a >> node's score depends on who evaluates it (eg the flow-based algorithm >> you sketched above) then how do we agree about which nodes to ignore? > > Well can we just use it from one node's perspective? When I last looked into > it that looked very risky. Do you mean every node calculates the trust thresholds from one node's perspective, or each node calculates them from its own perspective? Cheers, Michael From m.rogers at cs.ucl.ac.uk Fri Jan 4 00:42:01 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 00:42:01 +0000 Subject: [freenet-dev] Why getting rid of HTL is bad was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <200801040016.28579.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801030000.13355.toad@amphibian.dyndns.org> <477C3BEF.9090308@cs.ucl.ac.uk> <200801040016.28579.toad@amphibian.dyndns.org> Message-ID: <477D80D9.5090302@cs.ucl.ac.uk> Matthew Toseland wrote: >> We're using a reliable transport between nodes, so the only things that >> should cause a timeout are overloaded, crashed or faulty nodes. > > Not true. Very long paths can also cause this. Ah, good point. >> The closest-location-so-far tells an attacker something about the path a >> request has followed - among other things, it allows the attacker to >> rule out certain nodes the request definitely hasn't passed through. But >> how would we quantify that in terms of the size of the initiator's >> anonymity set or the probability that a given node is the initiator, for >> example? > > Well, how does the attacker rule out previous nodes? Any node that's closer to the target than the closest-location-so-far has never seen the request, otherwise the closest-location-so-far wouldn't have its current value. > How many can he rule out? That's kind of the point: I can't work out how to quantify how much the attacker can learn, which means it's possible that a clever attacker can learn a lot. Whereas with a weighted coin (despite its other disadvantages) at least we know how much the attacker can learn. Cheers, Michael From toad at amphibian.dyndns.org Fri Jan 4 00:44:15 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:44:15 +0000 Subject: [freenet-dev] Vulnerability in inserting the manifest before finishing In-Reply-To: <477C4357.6020805@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801030011.55484.toad@amphibian.dyndns.org> <477C4357.6020805@cs.ucl.ac.uk> Message-ID: <200801040044.15645.toad@amphibian.dyndns.org> On Thursday 03 January 2008 02:07, Michael Rogers wrote: > Matthew Toseland wrote: > >> There's a tradeoff here: large cells provide a large anonymity set if > >> the attacker's outside the cell, but they make it more likely that > >> there's an attacker inside the cell who can attack key distribution etc. > > > > Sure, but inside the cell it's harder to attack than outside it, surely that > > is the whole point? Tor, I2P, Mixmaster and other traditional onion routers > > (apart from spanning tree impl's) use a whole-network cell. > > Right, but they all fudge the key distribution problem. Tor and > Mixminion use central directory servers; I2P uses a DHT made of > untrusted nodes; Tarzan uses an insecure and unscalable gossip protocol; > MorphMix uses an insecure collusion detection mechanism [1]. > > I understand why premix routing is preferable to tunnels but you have to > solve the key distribution problem - your credibility score proposal > sounds promising but I'm not sure it's possible to make it Sybilproof > and still ensure that all nodes agree about the topology. Well, is it possible for each node to choose which nodes it trusts and still be secure? Last I looked you'd get a lot of oscillation problems, statistical attacks (proportion of requests - correlate with how many nodes are within the target's horizon etc). > > Cheers, > Michael > > [1] http://www.crhc.uiuc.edu/~nikita/papers/pet2006-morphmix.pdf -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/04bac2e7/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 00:47:57 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 00:47:57 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <477D7FBE.9020506@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040014.40597.toad@amphibian.dyndns.org> <477D7FBE.9020506@cs.ucl.ac.uk> Message-ID: <200801040047.57467.toad@amphibian.dyndns.org> On Friday 04 January 2008 00:37, Michael Rogers wrote: > Matthew Toseland wrote: > > The tunnel is constructed onion-style during the request's forwarding. The > > return tunnel likewise. There is no tunnel construction time. > > Ah cool, I see what you mean now. > > >> It doesn't matter that most of the samples are wrong - all > >> that matters is that the initiator is more likely to be recorded than a > >> random node (or a random member of the cell). > > > > Ok, this is possible, although slow. Is it fatal? How much data does the > > attacker need? > > Hard to say in general - it depends on an awful lot of variables. But we > know that increasing the number of linkable tunnels increases the number > of samples the attacker can collect. One classic strategy from mixmaster etc is to have a long delay at each step. We could do this for the more sensitive requests. The objective would be to send on the requests (more likely inserts) in groups big enough to likely contain requests from all the nodes in the cell. > > >> We're caught in a cleft stick here: algorithms that give a universally > >> agreed score for each node are vulnerable to Sybil attacks [1], but if a > >> node's score depends on who evaluates it (eg the flow-based algorithm > >> you sketched above) then how do we agree about which nodes to ignore? > > > > Well can we just use it from one node's perspective? When I last looked into > > it that looked very risky. > > Do you mean every node calculates the trust thresholds from one node's > perspective, or each node calculates them from its own perspective? How about a node can't join the cell unless its credibility with all or most other members is over some value? > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/29f1ad07/attachment.pgp From m.rogers at cs.ucl.ac.uk Fri Jan 4 00:50:47 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 00:50:47 +0000 Subject: [freenet-dev] Tunnels vs premix routing was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <200801040020.53673.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801030008.57919.toad@amphibian.dyndns.org> <477C3F20.7070208@cs.ucl.ac.uk> <200801040020.53673.toad@amphibian.dyndns.org> Message-ID: <477D82E7.9080809@cs.ucl.ac.uk> Matthew Toseland wrote: >> Right, but that's also the case at the moment: if requests travel n hops >> on average then there's a 1/n chance that the previous hop is the >> initiator. I don't see how you can get away from that without onion routing. > > Indeed. Which is why afaics we need onion routing. Would tunneling without > onion routing increase our vulnerability? It looks like it might to me. How would it increase our vulnerability? The attacker already knows there's a high probability that the previous hop is the initiator. But currently, the attacker gets one sample per request. With tunneling, the attacker would get one sample per tunnel. > Sure, and from time to time they will be DoSed and they will have to reroute. > Or they will route through a series of overloaded nodes and have to reroute. > Even here we end up with multiple tunnels. Yup, we can't avoid rebuilding failed tunnels, but as long as each tunnel carries more than one request on average it's still an improvement over the current situation (at least in terms of anonymity - obviously there's a cost in terms of hop count). > Yes but swapping already requires we reveal a lot of the topology, that's not > a big deal IMHO. The information in swap requests can be more or less anonymised, can't it? (Locations but no long-term node IDs, maybe a weighted coin instead of a hop counter?) > It partly depends on what you think about local attackers - > without premix routing, local attackers are extremely powerful. Tunnels should slow down local statistical attacks. Cheers, Michael From m.rogers at cs.ucl.ac.uk Fri Jan 4 01:16:37 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 01:16:37 +0000 Subject: [freenet-dev] Predecessor sampling and tunnel padding was Re: One tunnel per request: justification In-Reply-To: <200801040035.26492.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801022352.54143.toad@amphibian.dyndns.org> <477C379F.4050706@cs.ucl.ac.uk> <200801040035.26492.toad@amphibian.dyndns.org> Message-ID: <477D88F5.3000907@cs.ucl.ac.uk> Matthew Toseland wrote: > Okay, so it's a predecessor sample, a very noisy source, but it will > eventually reveal the originator if one of the attacker's nodes is connected > to it. And the frequency data will also eventually show nodes close to the > originator but not equal to it. Hmm, good point, I hadn't thought about indirectly identifying the initiator by identifying its neighbours. Could be useful if a darknet node has opennet neighbours... > So suppose we use long lived tunnels. Tagging on long-lived tunnels is > trivial. So we have a high quality sample on every new tunnel. It will be > essential to have multiple tunnels for adequate performance and reliability. Right, but the more parallel tunnels we use, the fewer requests each one carries and the harder it is to tag them. Somewhere between one-tunnel-per-request and one-tunnel-per-node is the optimal tradeoff between number of samples and quality of samples... but I have no idea how to find it. :-) The only rule of thumb I can come up with is that tunnels shouldn't link messages that would otherwise be unlinkable: two splitfiles shouldn't share a tunnel, two Frost IDs shouldn't share a tunnel, etc. > What is the impact of the node creating two parallel tunnels each through one > of its peer nodes which is run by the attacker? If he can link them, there is > still the possibility that they are both from intermediary nodes from another > originator - but this is probably less likely than the node being the > originator, so if we have any long term data we can catch him. If the routes are random, two tunnels that pass through different neighbours of X don't reveal any more information than two tunnels that pass through the same neighbour of X: the probability of the tunnels parting ways at X doesn't depend on whether X is the initiator. (I'll answer the tunnel padding proposal in another email.) Cheers, Michael From m.rogers at cs.ucl.ac.uk Fri Jan 4 01:59:56 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 01:59:56 +0000 Subject: [freenet-dev] Predecessor sampling and tunnel padding was Re: One tunnel per request: justification In-Reply-To: <200801040035.26492.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801022352.54143.toad@amphibian.dyndns.org> <477C379F.4050706@cs.ucl.ac.uk> <200801040035.26492.toad@amphibian.dyndns.org> Message-ID: <477D931C.5040107@cs.ucl.ac.uk> Matthew Toseland wrote: > If the internal MAC is invalid on a packet, the endpoint silently drops > the packet. I think I can get round it. All attacker-controlled nodes share a symmetric key. When an attacker-controlled node is asked to participate in a tunnel and it's not the endpoint, it injects a single packet into the tunnel, replacing a bogus packet if possible, otherwise replacing a non-bogus packet. The injected packet contains its predecessor's identity, and is encrypted and MACed with the attacker's key. When an attacker-controlled node is selected to be the endpoint of a tunnel, it looks for packets MACed with the attacker's key and decrypts them to collect predecessor samples. If a tunnel contains two non-adjacent attackers, one of which is the endpoint, the nodes between the attackers can't distinguish the injected packet from a genuine packet, so they pass it on. Cheers, Michael From m.rogers at cs.ucl.ac.uk Fri Jan 4 02:07:07 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 02:07:07 +0000 Subject: [freenet-dev] Vulnerability in inserting the manifest before finishing In-Reply-To: <200801040044.15645.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801030011.55484.toad@amphibian.dyndns.org> <477C4357.6020805@cs.ucl.ac.uk> <200801040044.15645.toad@amphibian.dyndns.org> Message-ID: <477D94CB.4090800@cs.ucl.ac.uk> Matthew Toseland wrote: > Well, is it possible for each node to choose which nodes it trusts and still > be secure? Last I looked you'd get a lot of oscillation problems, statistical > attacks (proportion of requests - correlate with how many nodes are within > the target's horizon etc). Yeah, it seems like there might be cases where, given a set of exit nodes of linkable tunnels, an attacker could narrow down the number of initiators who would trust all those exit nodes. All the required information is common knowledge because the neighbour lists are broadcast. Cheers, Michael From m.rogers at cs.ucl.ac.uk Fri Jan 4 02:18:48 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 02:18:48 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <200801040047.57467.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040014.40597.toad@amphibian.dyndns.org> <477D7FBE.9020506@cs.ucl.ac.uk> <200801040047.57467.toad@amphibian.dyndns.org> Message-ID: <477D9788.6070302@cs.ucl.ac.uk> Matthew Toseland wrote: > One classic strategy from mixmaster etc is to have a long delay at each step. > We could do this for the more sensitive requests. The objective would be to > send on the requests (more likely inserts) in groups big enough to likely > contain requests from all the nodes in the cell. Yup, you could definitely make timing attacks difficult if you're willing to pay the price in latency. Could be useful for things like Frost. >> Do you mean every node calculates the trust thresholds from one node's >> perspective, or each node calculates them from its own perspective? > > How about a node can't join the cell unless its credibility with all or most > other members is over some value? Sounds promising - but could that result in some nodes not being allowed to join any cell? Also, a node on one edge of the cell needs to be able to calculate trust scores from the point of view of a node on the opposite edge, so you have to spread the topology information twice as far. Cheers, Michael From toad at amphibian.dyndns.org Fri Jan 4 14:10:22 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 14:10:22 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <477D9788.6070302@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040047.57467.toad@amphibian.dyndns.org> <477D9788.6070302@cs.ucl.ac.uk> Message-ID: <200801041410.22489.toad@amphibian.dyndns.org> On Friday 04 January 2008 02:18, Michael Rogers wrote: > Matthew Toseland wrote: > > One classic strategy from mixmaster etc is to have a long delay at each step. > > We could do this for the more sensitive requests. The objective would be to > > send on the requests (more likely inserts) in groups big enough to likely > > contain requests from all the nodes in the cell. > > Yup, you could definitely make timing attacks difficult if you're > willing to pay the price in latency. Could be useful for things like Frost. I was thinking of long-term requests in general - large downloads for example. Do we need to do this on the return journey or only on the outward? > > >> Do you mean every node calculates the trust thresholds from one node's > >> perspective, or each node calculates them from its own perspective? > > > > How about a node can't join the cell unless its credibility with all or most > > other members is over some value? > > Sounds promising - but could that result in some nodes not being allowed > to join any cell? Yes. If you're a leaf node you can't use premix routing. > > Also, a node on one edge of the cell needs to be able to calculate trust > scores from the point of view of a node on the opposite edge, so you > have to spread the topology information twice as far. True. > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/d6f7c5fc/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 14:16:47 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 14:16:47 +0000 Subject: [freenet-dev] Predecessor sampling and tunnel padding was Re: One tunnel per request: justification In-Reply-To: <477D88F5.3000907@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040035.26492.toad@amphibian.dyndns.org> <477D88F5.3000907@cs.ucl.ac.uk> Message-ID: <200801041416.54969.toad@amphibian.dyndns.org> On Friday 04 January 2008 01:16, Michael Rogers wrote: > Matthew Toseland wrote: > > Okay, so it's a predecessor sample, a very noisy source, but it will > > eventually reveal the originator if one of the attacker's nodes is connected > > to it. And the frequency data will also eventually show nodes close to the > > originator but not equal to it. > > Hmm, good point, I hadn't thought about indirectly identifying the > initiator by identifying its neighbours. Could be useful if a darknet > node has opennet neighbours... Opennet nodes won't participate in premix routing. We may build something different, something more like I2P, for opennet nodes. Anyway, attacks involving moving closer to the target are a worry in some scenarios even on darknet - they're something we want to minimize. > > > So suppose we use long lived tunnels. Tagging on long-lived tunnels is > > trivial. So we have a high quality sample on every new tunnel. It will be > > essential to have multiple tunnels for adequate performance and reliability. > > Right, but the more parallel tunnels we use, the fewer requests each one > carries and the harder it is to tag them. Somewhere between > one-tunnel-per-request and one-tunnel-per-node is the optimal tradeoff > between number of samples and quality of samples... but I have no idea > how to find it. :-) Okay, so on the one hand, more requests per tunnel means easier tagging (noisier samples). On the other hand, less requests per tunnel means more tunnels and more predecessor samples (more samples). Hmm... I suppose finding the optimal number of requests per tunnel is a matter for simulation? Would it be hideously complicated? > > The only rule of thumb I can come up with is that tunnels shouldn't link > messages that would otherwise be unlinkable: two splitfiles shouldn't > share a tunnel, two Frost IDs shouldn't share a tunnel, etc. Well yeah but Frost IDs and splitfiles, and splitfiles and splitfiles, often are linked. So it's not easy. > > > What is the impact of the node creating two parallel tunnels each through one > > of its peer nodes which is run by the attacker? If he can link them, there is > > still the possibility that they are both from intermediary nodes from another > > originator - but this is probably less likely than the node being the > > originator, so if we have any long term data we can catch him. > > If the routes are random, two tunnels that pass through different > neighbours of X don't reveal any more information than two tunnels that > pass through the same neighbour of X: the probability of the tunnels > parting ways at X doesn't depend on whether X is the initiator. It doesn't? I had assumed they always parted ways at the beginning... hmmm. Can you elaborate a little? > > (I'll answer the tunnel padding proposal in another email.) > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/2c9fc452/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 14:20:18 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 14:20:18 +0000 Subject: [freenet-dev] Predecessor sampling and tunnel padding was Re: One tunnel per request: justification In-Reply-To: <477D931C.5040107@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040035.26492.toad@amphibian.dyndns.org> <477D931C.5040107@cs.ucl.ac.uk> Message-ID: <200801041420.18681.toad@amphibian.dyndns.org> On Friday 04 January 2008 01:59, Michael Rogers wrote: > Matthew Toseland wrote: > > If the internal MAC is invalid on a packet, the endpoint silently drops > > the packet. > > I think I can get round it. > > All attacker-controlled nodes share a symmetric key. When an > attacker-controlled node is asked to participate in a tunnel and it's > not the endpoint, it injects a single packet into the tunnel, replacing > a bogus packet if possible, otherwise replacing a non-bogus packet. The > injected packet contains its predecessor's identity, and is encrypted > and MACed with the attacker's key. > > When an attacker-controlled node is selected to be the endpoint of a > tunnel, it looks for packets MACed with the attacker's key and decrypts > them to collect predecessor samples. > > If a tunnel contains two non-adjacent attackers, one of which is the > endpoint, the nodes between the attackers can't distinguish the injected > packet from a genuine packet, so they pass it on. Doh. Okay, so tunnel padding remains an unsolved and perhaps insoluble problem. > > Cheers, > Michael > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/1f1eb1fd/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 14:32:49 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 14:32:49 +0000 Subject: [freenet-dev] Tunnels vs premix routing was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <477D82E7.9080809@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040020.53673.toad@amphibian.dyndns.org> <477D82E7.9080809@cs.ucl.ac.uk> Message-ID: <200801041432.54681.toad@amphibian.dyndns.org> On Friday 04 January 2008 00:50, Michael Rogers wrote: > Matthew Toseland wrote: > >> Right, but that's also the case at the moment: if requests travel n hops > >> on average then there's a 1/n chance that the previous hop is the > >> initiator. I don't see how you can get away from that without onion routing. > > > > Indeed. Which is why afaics we need onion routing. Would tunneling without > > onion routing increase our vulnerability? It looks like it might to me. > > How would it increase our vulnerability? The attacker already knows > there's a high probability that the previous hop is the initiator. But > currently, the attacker gets one sample per request. With tunneling, the > attacker would get one sample per tunnel. Hmm, maybe you are right. There are two things that potentially increase our vulnerability: - The tunnel length is likely to be shorter than the routing length. So P(predecessor = originator) drops from 1/routing length to 1/tunnel length - say 1 in 15 down to 1 in 5. Routing length is very long at the moment for failed requests, maybe too long because of perverse topology. So you need fewer samples. - The attacker can link two tunnels with a greater confidence than two requests. Right? So again you need fewer samples. On the other hand, you would have a *lot* fewer samples to work with than with requests. So probably it is better. Hmmm... I had thought that if the attacker sees two tunnels for the same request out of the same node that he can be certain that is the originator. But this isn't really true, if the tunnels are random. > > > Sure, and from time to time they will be DoSed and they will have to reroute. > > Or they will route through a series of overloaded nodes and have to reroute. > > Even here we end up with multiple tunnels. > > Yup, we can't avoid rebuilding failed tunnels, but as long as each > tunnel carries more than one request on average it's still an > improvement over the current situation (at least in terms of anonymity - > obviously there's a cost in terms of hop count). > > > Yes but swapping already requires we reveal a lot of the topology, that's not > > a big deal IMHO. > > The information in swap requests can be more or less anonymised, can't > it? (Locations but no long-term node IDs, maybe a weighted coin instead > of a hop counter?) Yes, but a clever attacker can reconstruct the topology for a few hops without too much difficulty. It's messy, I haven't come up with a good algorithm yet, but I haven't really tried: there should be one. > > > It partly depends on what you think about local attackers - > > without premix routing, local attackers are extremely powerful. > > Tunnels should slow down local statistical attacks. > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/cc391d84/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 14:34:20 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 14:34:20 +0000 Subject: [freenet-dev] Vulnerability in inserting the manifest before finishing In-Reply-To: <477D94CB.4090800@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040044.15645.toad@amphibian.dyndns.org> <477D94CB.4090800@cs.ucl.ac.uk> Message-ID: <200801041434.20713.toad@amphibian.dyndns.org> On Friday 04 January 2008 02:07, Michael Rogers wrote: > Matthew Toseland wrote: > > Well, is it possible for each node to choose which nodes it trusts and still > > be secure? Last I looked you'd get a lot of oscillation problems, statistical > > attacks (proportion of requests - correlate with how many nodes are within > > the target's horizon etc). > > Yeah, it seems like there might be cases where, given a set of exit > nodes of linkable tunnels, an attacker could narrow down the number of > initiators who would trust all those exit nodes. All the required > information is common knowledge because the neighbour lists are broadcast. Right, even if each node has the same number of exit nodes. And then you've got the effects of nodes going down or going up on the set of exit nodes for each node. You need to have a lot of inertia here, or that'll give away a lot of information. Hence my coming up with the idea of cells. The question is, is it workable? > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/4448f20f/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 14:53:01 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 14:53:01 +0000 Subject: [freenet-dev] [freenet-cvs] r16871 - trunk/freenet/src/freenet/node In-Reply-To: <20080104015340.9043247BBD6@freenetproject.org> References: <20080104015340.9043247BBD6@freenetproject.org> Message-ID: <200801041453.01460.toad@amphibian.dyndns.org> location > 1.0 is possible if the node set it too high and told us, no? On Friday 04 January 2008 01:53, you wrote: > Author: robert > Date: 2008-01-04 01:53:40 +0000 (Fri, 04 Jan 2008) > New Revision: 16871 > > Modified: > trunk/freenet/src/freenet/node/PeerNode.java > Log: > comments > > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-04 00:27:19 UTC (rev 16870) > +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-04 01:53:40 UTC (rev 16871) > @@ -81,13 +81,15 @@ > public abstract class PeerNode implements PeerContext, USKRetrieverCallback { > > private String lastGoodVersion; > - /** Set to true based on a relevant incoming handshake from this peer > - * Set true if this peer has a incompatible older build than we are > - */ > + /** > + * True if this peer has a build number older than our last-known-good build number. > + * Note that even if this is true, the node can still be 'connected'. > + */ > protected boolean unroutableOlderVersion; > - /** Set to true based on a relevant incoming handshake from this peer > - * Set true if this peer has a incompatible newer build than we are > - */ > + /** > + * True if this peer reports that our build number is before their last-known-good build number. > + * Note that even if this is true, the node can still be 'connected'. > + */ > protected boolean unroutableNewerVersion; > protected boolean disableRouting; > protected boolean disableRoutingHasBeenSetLocally; > @@ -158,7 +160,7 @@ > private int handshakeCount; > /** After this many failed handshakes, we start the ARK fetcher. */ > private static final int MAX_HANDSHAKE_COUNT = 2; > - /** Current location in the keyspace */ > + /** Current location in the keyspace, or -1 if it is unknown */ > private double currentLocation; > /** Node identity; for now a block of data, in future a > * public key (FIXME). Cannot be changed. > @@ -850,47 +852,54 @@ > } > > /** > - * What is my current keyspace location? > + * Returns this peer's current keyspace location, or -1 if it is unknown. > */ > public synchronized double getLocation() { > return currentLocation; > } > > /** > - * Returns a unique node identifier (usefull to compare 2 pn) > + * Returns a unique node identifier (usefull to compare two peernodes). > */ > public int getIdentityHash() { > return hashCode; > } > > /** > - * Is this peer too old for us? (i.e. our lastGoodVersion is newer than it's version) > - * > - */ > + * Returns true if the last-known build number for this peer is to old to allow traffic to be routed to it. > + * This does not give any indication as to the connection status of the peer. > + */ > public synchronized boolean isUnroutableOlderVersion() { > return unroutableOlderVersion; > } > > /** > - * Is this peer too new for us? (i.e. our version is older than it's lastGoodVersion) > - * > - */ > + * Returns true if this (or another) peer has reported to us that our build number is too old for data to be routed > + * to us. In turn, we will not route data to them either. Does not strictly indicate that the peer is connected. > + */ > public synchronized boolean isUnroutableNewerVersion() { > return unroutableNewerVersion; > } > > /** > - * Is this peer currently connected? (And routing-compatible, i.e. can we route > - * requests to it, ignoring backoff) > + * Returns true if requests can be routed through this peer. True if the peer's location is known, presently > + * connected, and routing-compatible. That is, ignoring backoff, the peer's location is known, build number > + * is compatible, and routing has not been explicitly disabled. > * > * Note possible deadlocks! PeerManager calls this, we call > * PeerManager in e.g. verified. > */ > public boolean isRoutable() { > + //FIXME: isConnected() is redundant if 'isRoutable', right? ... currentLocation>1.0 is impossible. > return isConnected() && isRoutingCompatible() && > !(currentLocation < 0.0 || currentLocation > 1.0); > } > > + /** > + * Returns true if (apart from actually knowing the peer's location), it is presumed that this peer could route requests. > + * True if this peer's build number is not 'too-old' or 'too-new', actively connected, and not marked as explicity disabled. > + * Does not reflect any 'backoff' logic. > + */ > public boolean isRoutingCompatible() { > long now = System.currentTimeMillis(); > synchronized(this) { > @@ -1220,7 +1229,7 @@ > synchronized(this) { > long delay; > if(unroutableOlderVersion || unroutableNewerVersion || disableRouting) { > - // Let them know we're here, but have no hope of connecting > + // Let them know we're here, but have no hope of routing general data to them. > delay = Node.MIN_TIME_BETWEEN_VERSION_SENDS + node.random.nextInt(Node.RANDOMIZED_TIME_BETWEEN_VERSION_SENDS); > } else if(invalidVersion() && !firstHandshake) { > delay = Node.MIN_TIME_BETWEEN_VERSION_PROBES + node.random.nextInt(Node.RANDOMIZED_TIME_BETWEEN_VERSION_PROBES); > @@ -1802,6 +1811,7 @@ > Message dRouting = DMT.createRoutingStatus(!disableRoutingHasBeenSetLocally); > > try { > + //FIXME: Why is location only if routable, and the others not? > if(isRoutable()) > sendAsync(locMsg, null, 0, null); > sendAsync(ipMsg, null, 0, null); > @@ -1896,10 +1906,16 @@ > return !Version.checkArbitraryGoodVersion(Version.getVersionString(), lastGoodVersion); > } > > + /** > + * The same as isUnroutableOlderVersion, but not synchronized. > + */ > public boolean publicInvalidVersion() { > return unroutableOlderVersion; > } > > + /** > + * The same as inUnroutableNewerVersion. > + */ > public synchronized boolean publicReverseInvalidVersion() { > return unroutableNewerVersion; > } > @@ -2013,7 +2029,7 @@ > currentLocation = newLoc; > } > } catch(FSParseException e) { > - // Location is optional, we will wait for FNPLocChangeNotification > + // Location is optional, we will wait for FNPLocChangeNotification. Until then we will use the last known location (or -1 if we have never known). > if(logMINOR) > Logger.minor(this, "Invalid or null location, waiting for FNPLocChangeNotification: " + e); > } > @@ -2780,18 +2796,21 @@ > return handshakeCount; > } > > + /** > + * Queries the Version class to determine if this peers advertised build-number is either too-old or > + * to new for the routing of requests. > + */ > synchronized void updateVersionRoutablity() { > unroutableOlderVersion = forwardInvalidVersion(); > unroutableNewerVersion = reverseInvalidVersion(); > } > > /** > - * Has the node gone from being routable to being non-routable? > - * This will return true if our lastGoodBuild has changed due to a timed mandatory. > + * Will return true if routing to this node is either explictly disabled, or disabled due to > + * noted incompatiblity in build-version numbers. > + * Logically: "not(isRoutable())", but will return false even if disconnected (meaning routing is not disabled). > */ > public synchronized boolean noLongerRoutable() { > - // TODO: We should disconnect here if "protocol version mismatch", maybe throwing an exception > - // TODO: shouldDisconnectNow() is hopefully only called when we're connected, otherwise we're breaking the meaning of verifiedIncompable[Older| Newer]Version > if(unroutableNewerVersion || unroutableOlderVersion || disableRouting) > return true; > return false; > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/10aec774/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 14:58:04 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 14:58:04 +0000 Subject: [freenet-dev] [freenet-cvs] r16875 - trunk/freenet/src/freenet/node/fcp In-Reply-To: <20080104034344.A040947A0B2@freenetproject.org> References: <20080104034344.A040947A0B2@freenetproject.org> Message-ID: <200801041458.05306.toad@amphibian.dyndns.org> On Friday 04 January 2008 03:43, you wrote: > Author: robert > Date: 2008-01-04 03:43:44 +0000 (Fri, 04 Jan 2008) > New Revision: 16875 > > Modified: > trunk/freenet/src/freenet/node/fcp/ClientGet.java > trunk/freenet/src/freenet/node/fcp/ClientPut.java > trunk/freenet/src/freenet/node/fcp/ClientRequest.java > Log: > always set start=true if finished==true (bug#1962), persist 'finished' for put requests (?!) Don't do that (the latter). It's set already in super.toFieldSet(). The code will now throw because it is set twice. I'll revert it. > > Modified: trunk/freenet/src/freenet/node/fcp/ClientGet.java > =================================================================== > --- trunk/freenet/src/freenet/node/fcp/ClientGet.java 2008-01-04 03:18:41 UTC (rev 16874) > +++ trunk/freenet/src/freenet/node/fcp/ClientGet.java 2008-01-04 03:43:44 UTC (rev 16875) > @@ -308,12 +308,8 @@ > client.queueClientRequestMessage(msg, 0); > } > > - if(finished){ > - if(succeeded) > + if(finished && succeeded) > allDataPending = new AllDataMessage(returnBucket, identifier, global, startupTime, completionTime); > - else > - started = true; > - } > } > > public void start() { > > Modified: trunk/freenet/src/freenet/node/fcp/ClientPut.java > =================================================================== > --- trunk/freenet/src/freenet/node/fcp/ClientPut.java 2008-01-04 03:18:41 UTC (rev 16874) > +++ trunk/freenet/src/freenet/node/fcp/ClientPut.java 2008-01-04 03:43:44 UTC (rev 16875) > @@ -411,6 +411,7 @@ > fs.putSingle("TargetFilename", targetFilename); > fs.putSingle("EarlyEncode", Boolean.toString(earlyEncode)); > fs.put("BinaryBlob", binaryBlob); > + fs.putSingle("Finished", Boolean.toString(finished)); > > return fs; > } > > Modified: trunk/freenet/src/freenet/node/fcp/ClientRequest.java > =================================================================== > --- trunk/freenet/src/freenet/node/fcp/ClientRequest.java 2008-01-04 03:18:41 UTC (rev 16874) > +++ trunk/freenet/src/freenet/node/fcp/ClientRequest.java 2008-01-04 03:43:44 UTC (rev 16875) > @@ -107,6 +107,8 @@ > final String stime = fs.get("StartupTime"); > this.startupTime = stime == null ? System.currentTimeMillis() : Fields.parseLong(stime); > completionTime = fs.getLong("CompletionTime", 0); > + if (finished) > + started=true; > } > > /** Lost connection */ > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/40ddfe06/attachment.pgp From robert at emu.freenetproject.org Fri Jan 4 16:23:33 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 4 Jan 2008 10:23:33 -0600 Subject: [freenet-dev] [freenet-cvs] r16871 - trunk/freenet/src/freenet/node In-Reply-To: <200801041453.01460.toad@amphibian.dyndns.org> References: <20080104015340.9043247BBD6@freenetproject.org> <200801041453.01460.toad@amphibian.dyndns.org> Message-ID: <928224AE-CE8C-4136-AC84-2CA14B5EA29A@emu.freenetproject.org> On Jan 4, 2008, at 8:53 AM, Matthew Toseland wrote: > location > 1.0 is possible if the node set it too high and told us, > no? No... AFAICS currentLocation is private, the only time it is not set to a constant (-1) is from "Location.getLocation(locationString);" or "updateLocation(double newLoc);". updateLocation() has an explicit guard against setting it beyond 0&1, and getLocation() will throw a parse exception if it is beyond 0&1. In fact, rather than setting currentLocation to -1, we could probably set isRoutable=false and it would be even simpler. -- Robert Hailey > On Friday 04 January 2008 01:53, you wrote: >> Author: robert >> Date: 2008-01-04 01:53:40 +0000 (Fri, 04 Jan 2008) >> New Revision: 16871 >> >> Modified: >> trunk/freenet/src/freenet/node/PeerNode.java >> Log: >> comments >> >> >> Modified: trunk/freenet/src/freenet/node/PeerNode.java >> =================================================================== >> [...] >> public boolean isRoutable() { >> + //FIXME: isConnected() is redundant if 'isRoutable', right? ... >> currentLocation>1.0 is impossible. >> return isConnected() && isRoutingCompatible() && >> !(currentLocation < 0.0 || currentLocation > 1.0); >> } >> -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080104/8757886f/attachment.htm From robert at emu.freenetproject.org Fri Jan 4 16:25:58 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 4 Jan 2008 10:25:58 -0600 Subject: [freenet-dev] [freenet-cvs] r16875 - trunk/freenet/src/freenet/node/fcp In-Reply-To: <200801041458.05306.toad@amphibian.dyndns.org> References: <20080104034344.A040947A0B2@freenetproject.org> <200801041458.05306.toad@amphibian.dyndns.org> Message-ID: On Jan 4, 2008, at 8:58 AM, Matthew Toseland wrote: > On Friday 04 January 2008 03:43, you wrote: >> Author: robert >> Date: 2008-01-04 03:43:44 +0000 (Fri, 04 Jan 2008) >> New Revision: 16875 >> >> Modified: >> trunk/freenet/src/freenet/node/fcp/ClientGet.java >> trunk/freenet/src/freenet/node/fcp/ClientPut.java >> trunk/freenet/src/freenet/node/fcp/ClientRequest.java >> Log: >> always set start=true if finished==true (bug#1962), persist >> 'finished' for > put requests (?!) > > Don't do that (the latter). It's set already in super.toFieldSet(). > The code > will now throw because it is set twice. I'll revert it. Thanks. I kept looking between ClientRequest and ClientPut trying to figure out how in the world it was presently working (or if it was). Now I see that unlike ClientGet, ClientPut extends an intermediate class (ClientPutBase). -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080104/6ebb6dff/attachment.htm From robert at emu.freenetproject.org Fri Jan 4 18:32:55 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 4 Jan 2008 12:32:55 -0600 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <20080104182211.0332D390A59@freenetproject.org> References: <20080104182211.0332D390A59@freenetproject.org> Message-ID: <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> Apparently until this revision 16886, (so long as any one node does not timeout) a node will take as long as necessary to exhaust routable peers. Even long after the original requestor has given up on that node. I suspect good and bad from this. On the one hand, all nodes will become much less busy (increase throughput, decrease latency), but the data may be not be found at all (as presently the node may continue to search all its peers and it be cached for next time). What do you think? -- Robert Hailey On Jan 4, 2008, at 12:22 PM, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-04 18:22:10 +0000 (Fri, 04 Jan 2008) > New Revision: 16886 > > Modified: > trunk/freenet/src/freenet/node/RequestSender.java > Log: > do not continue a search which has already locally timed out > > > Modified: trunk/freenet/src/freenet/node/RequestSender.java > =================================================================== > --- trunk/freenet/src/freenet/node/RequestSender.java 2008-01-04 > 17:22:34 UTC (rev 16885) > +++ trunk/freenet/src/freenet/node/RequestSender.java 2008-01-04 > 18:22:10 UTC (rev 16886) > @@ -136,6 +136,7 @@ > pubKey = ((NodeSSK)key).getPubKey(); > } > > + long startTime=System.currentTimeMillis(); > HashSet nodesRoutedTo = new HashSet(); > HashSet nodesNotIgnored = new HashSet(); > while(true) { > @@ -145,6 +146,12 @@ > finish(DATA_NOT_FOUND, null); > return; > } > + > + if (source!=null && System.currentTimeMillis()- > startTime>FETCH_TIMEOUT) { > + Logger.error(this, "discontinuing non-local request search, > general timeout"); > + finish(TIMED_OUT, null); > + return; > + } > > // Route it > PeerNode next; > @@ -586,6 +593,7 @@ > */ > public synchronized short waitUntilStatusChange(short mask) { > if(mask == WAIT_ALL) throw new IllegalArgumentException("Cannot > ignore all!"); > + long startTime=System.currentTimeMillis(); > while(true) { > short current = mask; // If any bits are set already, we > ignore those states. > > @@ -605,6 +613,10 @@ > } catch (InterruptedException e) { > // Ignore > } > + if (source!=null && System.currentTimeMillis()-startTime > > 2*FETCH_TIMEOUT) { > + Logger.error(this, "spending way too long waiting for request > sender to finish"); > + throw new RuntimeException("something is broken"); > + } > } > } > > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > From m.rogers at cs.ucl.ac.uk Fri Jan 4 18:42:37 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 18:42:37 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <200801041410.22489.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040047.57467.toad@amphibian.dyndns.org> <477D9788.6070302@cs.ucl.ac.uk> <200801041410.22489.toad@amphibian.dyndns.org> Message-ID: <477E7E1D.9060300@cs.ucl.ac.uk> Matthew Toseland wrote: > I was thinking of long-term requests in general - large downloads for example. > > Do we need to do this on the return journey or only on the outward? Both, otherwise the nodes in the return tunnel can gather successor samples. >> Sounds promising - but could that result in some nodes not being allowed >> to join any cell? > > Yes. If you're a leaf node you can't use premix routing. Not just leaves, but perhaps nodes whose neighbours all belong to different clusters? Cheers, Michael From m.rogers at cs.ucl.ac.uk Fri Jan 4 18:55:13 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 18:55:13 +0000 Subject: [freenet-dev] Predecessor sampling and tunnel padding was Re: One tunnel per request: justification In-Reply-To: <200801041416.54969.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040035.26492.toad@amphibian.dyndns.org> <477D88F5.3000907@cs.ucl.ac.uk> <200801041416.54969.toad@amphibian.dyndns.org> Message-ID: <477E8111.2010703@cs.ucl.ac.uk> Matthew Toseland wrote: > Opennet nodes won't participate in premix routing. What happens if the darknet nodes don't form a connected subnetwork? > Okay, so on the one hand, more requests per tunnel means easier tagging > (noisier samples). On the other hand, less requests per tunnel means more > tunnels and more predecessor samples (more samples). Hmm... I suppose finding > the optimal number of requests per tunnel is a matter for simulation? Would > it be hideously complicated? As always, the problem is making it realistic... we can do a simplified simulation but it might not capture some relevant feature (like daily/weekly uptime patterns, just as an example of something that's likely to be relevant but hard to simulate realistically). > Well yeah but Frost IDs and splitfiles, and splitfiles and splitfiles, often > are linked. So it's not easy. Absolutely - we'd probably need to rely on clients to make informed choices (eg allow an arbitrary "batch ID" to be associated with each request, and try to send requests with the same batch ID through the same tunnel or set of tunnels, and requests with different batch IDs through different tunnels). >> If the routes are random, two tunnels that pass through different >> neighbours of X don't reveal any more information than two tunnels that >> pass through the same neighbour of X: the probability of the tunnels >> parting ways at X doesn't depend on whether X is the initiator. > > It doesn't? I had assumed they always parted ways at the beginning... hmmm. > Can you elaborate a little? If each hop of each tunnel is chosen independently and randomly, then once two tunnels reach X, the probability that their next hop is the same doesn't depend on the route so far - so it's the same as the probability of two tunnels that started at X having the same next hop. Cheers, Michael From m.rogers at cs.ucl.ac.uk Fri Jan 4 19:35:19 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 19:35:19 +0000 Subject: [freenet-dev] Tunnels vs premix routing was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <200801041432.54681.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040020.53673.toad@amphibian.dyndns.org> <477D82E7.9080809@cs.ucl.ac.uk> <200801041432.54681.toad@amphibian.dyndns.org> Message-ID: <477E8A77.5090307@cs.ucl.ac.uk> Matthew Toseland wrote: > - The tunnel length is likely to be shorter than the routing length. So > P(predecessor = originator) drops from 1/routing length to 1/tunnel length - > say 1 in 15 down to 1 in 5. True, the tunnels would have to be quite long. On the other hand we don't know how much information the hop counter and closest-location-so-far are currently revealing, so it's possible an attacker has a better than 1 in 15 chance of identifying the initiator during the routing phase. (For example, if the hop counter isn't MAX or MAX-1 then the previous hop can't be the initiator, right?) > Yes, but a clever attacker can reconstruct the topology for a few hops without > too much difficulty. It's messy, I haven't come up with a good algorithm yet, > but I haven't really tried: there should be one. Good point, we should be pessimistic. Cheers, Michael From toad at amphibian.dyndns.org Fri Jan 4 19:55:09 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 19:55:09 +0000 Subject: [freenet-dev] [freenet-cvs] r16871 - trunk/freenet/src/freenet/node In-Reply-To: <928224AE-CE8C-4136-AC84-2CA14B5EA29A@emu.freenetproject.org> References: <20080104015340.9043247BBD6@freenetproject.org> <200801041453.01460.toad@amphibian.dyndns.org> <928224AE-CE8C-4136-AC84-2CA14B5EA29A@emu.freenetproject.org> Message-ID: <200801041955.14447.toad@amphibian.dyndns.org> On Friday 04 January 2008 16:23, Robert Hailey wrote: > > On Jan 4, 2008, at 8:53 AM, Matthew Toseland wrote: > > location > 1.0 is possible if the node set it too high and told us, > > no? > > No... AFAICS currentLocation is private, the only time it is not set > to a constant (-1) is from "Location.getLocation(locationString);" or > "updateLocation(double newLoc);". > > updateLocation() has an explicit guard against setting it beyond 0&1, > and getLocation() will throw a parse exception if it is beyond 0&1. > > In fact, rather than setting currentLocation to -1, we could probably > set isRoutable=false and it would be even simpler. Good idea - but won't it get reset? -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/5c5bfb97/attachment.pgp From m.rogers at cs.ucl.ac.uk Fri Jan 4 19:57:50 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 04 Jan 2008 19:57:50 +0000 Subject: [freenet-dev] Vulnerability in inserting the manifest before finishing In-Reply-To: <200801041434.20713.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801040044.15645.toad@amphibian.dyndns.org> <477D94CB.4090800@cs.ucl.ac.uk> <200801041434.20713.toad@amphibian.dyndns.org> Message-ID: <477E8FBE.5080209@cs.ucl.ac.uk> Matthew Toseland wrote: > Right, even if each node has the same number of exit nodes. And then you've > got the effects of nodes going down or going up on the set of exit nodes for > each node. You need to have a lot of inertia here, or that'll give away a lot > of information. Hence my coming up with the idea of cells. The question is, > is it workable? If the members of a cell can somehow agree on its membership then I don't think it's a problem if exit nodes come and go (except in the general sense that if a node's ever offline when your pseudonym is active then that node can be removed from your anonymity set). But what happens if a cell is split due to nodes being offline? How can the members of a cell agree on its membership without including Sybil nodes? Start by finding a clique of n nodes (n-1 of your neighbours are all connected to each other). Recursively add every node that has at least n-1 neighbours in the cell. (Smaller values of n will result in more and larger cells.) Does the result depend on the order in which nodes are considered? Cheers, Michael From toad at amphibian.dyndns.org Fri Jan 4 20:30:21 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 20:30:21 +0000 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> References: <20080104182211.0332D390A59@freenetproject.org> <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> Message-ID: <200801042030.26315.toad@amphibian.dyndns.org> On Friday 04 January 2008 18:32, Robert Hailey wrote: > > Apparently until this revision 16886, (so long as any one node does > not timeout) a node will take as long as necessary to exhaust routable > peers. Even long after the original requestor has given up on that node. Yes. Is this bad? Obviously there are limits - if it gets a post-accepted timeout on any one node it will finish the request. Generally I think this is probably a good thing - the data is wanted, so why not find it? It will be cached and will be transferred later. With ULPRs it will even be transferred when we complete, despite the timeout. > > I suspect good and bad from this. On the one hand, all nodes will > become much less busy (increase throughput, decrease latency), but the > data may be not be found at all (as presently the node may continue to > search all its peers and it be cached for next time). Right, the advantage of the current system is that the data will probably be found eventually, and the next time the request is made it will be available before it times out. But if it's hard to find, if we timeout as you suggest, it may not be found. The only real reason I can think of to timeout as you do below would be for load balancing fairness: If you make a request, you had better be prepared (bandwidth wise) to accept the resulting data, if you're not, that's a denial of service attack. Load management relies on propagating the load back to the requestor. But the requestor can have only a very limited influence on timing out here so I don't consider it to be valid. > > What do you think? > > -- > Robert Hailey > > On Jan 4, 2008, at 12:22 PM, robert at freenetproject.org wrote: > > > Author: robert > > Date: 2008-01-04 18:22:10 +0000 (Fri, 04 Jan 2008) > > New Revision: 16886 > > > > Modified: > > trunk/freenet/src/freenet/node/RequestSender.java > > Log: > > do not continue a search which has already locally timed out > > > > > > Modified: trunk/freenet/src/freenet/node/RequestSender.java > > =================================================================== > > --- trunk/freenet/src/freenet/node/RequestSender.java 2008-01-04 > > 17:22:34 UTC (rev 16885) > > +++ trunk/freenet/src/freenet/node/RequestSender.java 2008-01-04 > > 18:22:10 UTC (rev 16886) > > @@ -136,6 +136,7 @@ > > pubKey = ((NodeSSK)key).getPubKey(); > > } > > > > + long startTime=System.currentTimeMillis(); > > HashSet nodesRoutedTo = new HashSet(); > > HashSet nodesNotIgnored = new HashSet(); > > while(true) { > > @@ -145,6 +146,12 @@ > > finish(DATA_NOT_FOUND, null); > > return; > > } > > + > > + if (source!=null && System.currentTimeMillis()- > > startTime>FETCH_TIMEOUT) { > > + Logger.error(this, "discontinuing non-local request search, > > general timeout"); > > + finish(TIMED_OUT, null); > > + return; > > + } > > > > // Route it > > PeerNode next; > > @@ -586,6 +593,7 @@ > > */ > > public synchronized short waitUntilStatusChange(short mask) { > > if(mask == WAIT_ALL) throw new IllegalArgumentException("Cannot > > ignore all!"); > > + long startTime=System.currentTimeMillis(); > > while(true) { > > short current = mask; // If any bits are set already, we > > ignore those states. > > > > @@ -605,6 +613,10 @@ > > } catch (InterruptedException e) { > > // Ignore > > } > > + if (source!=null && System.currentTimeMillis()-startTime > > > 2*FETCH_TIMEOUT) { > > + Logger.error(this, "spending way too long waiting for request > > sender to finish"); > > + throw new RuntimeException("something is broken"); > > + } > > } > > } > > > > > > _______________________________________________ > > cvs mailing list > > cvs at freenetproject.org > > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/d450f812/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 20:34:34 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 20:34:34 +0000 Subject: [freenet-dev] Vulnerability in inserting the manifest before finishing In-Reply-To: <477E8FBE.5080209@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801041434.20713.toad@amphibian.dyndns.org> <477E8FBE.5080209@cs.ucl.ac.uk> Message-ID: <200801042034.34556.toad@amphibian.dyndns.org> On Friday 04 January 2008 19:57, Michael Rogers wrote: > Matthew Toseland wrote: > > Right, even if each node has the same number of exit nodes. And then you've > > got the effects of nodes going down or going up on the set of exit nodes for > > each node. You need to have a lot of inertia here, or that'll give away a lot > > of information. Hence my coming up with the idea of cells. The question is, > > is it workable? > > If the members of a cell can somehow agree on its membership then I > don't think it's a problem if exit nodes come and go (except in the > general sense that if a node's ever offline when your pseudonym is > active then that node can be removed from your anonymity set). But what > happens if a cell is split due to nodes being offline? Above I was talking about without cells. But this is why it is important for a cell to be well-connected! Having said that, is going through untrusted nodes as gateways from one trusted node to another a problem? As long as we can choose two cell members to go through, we should be okay? > > How can the members of a cell agree on its membership without including > Sybil nodes? > > Start by finding a clique of n nodes (n-1 of your neighbours are all > connected to each other). Recursively add every node that has at least > n-1 neighbours in the cell. (Smaller values of n will result in more and > larger cells.) > > Does the result depend on the order in which nodes are considered? I would have thought so. > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/30ac74f1/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 4 20:39:50 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 4 Jan 2008 20:39:50 +0000 Subject: [freenet-dev] Tunnels vs premix routing was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <477E8A77.5090307@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801041432.54681.toad@amphibian.dyndns.org> <477E8A77.5090307@cs.ucl.ac.uk> Message-ID: <200801042039.50495.toad@amphibian.dyndns.org> On Friday 04 January 2008 19:35, Michael Rogers wrote: > Matthew Toseland wrote: > > - The tunnel length is likely to be shorter than the routing length. So > > P(predecessor = originator) drops from 1/routing length to 1/tunnel length - > > say 1 in 15 down to 1 in 5. > > True, the tunnels would have to be quite long. On the other hand we > don't know how much information the hop counter and > closest-location-so-far are currently revealing, so it's possible an > attacker has a better than 1 in 15 chance of identifying the initiator > during the routing phase. (For example, if the hop counter isn't MAX or > MAX-1 then the previous hop can't be the initiator, right?) Hmmm. That's true. So it may even be an improvement. Probe data can show how often we reset, but lets assume we reset 3 times (this is an ideal, in practice IIRC we reset more often than that). The 50% chance of decrementing at 10 means that 3 resets equals 6 nodes with HTL 10... When an attacker gets a tunneled request, he knows 1) there is a 20% chance that the node is the originator, and 2) that he is close to the originator, because he is early on the route: the tunnel is only active at the beginning. For a regular request, the HTL can be 10 much later on in the request. In which case the predecessor sample is useless, and the key is likely close to the location of the attacker. Is this a plus or a minus for tunnels? Somewhere on this thread you said that the probability of two related tunnels coming from the same node is independant of that node being the originator. I don't understand that. > > > Yes, but a clever attacker can reconstruct the topology for a few hops without > > too much difficulty. It's messy, I haven't come up with a good algorithm yet, > > but I haven't really tried: there should be one. > > Good point, we should be pessimistic. > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080104/453dfdea/attachment.pgp From robert at emu.freenetproject.org Fri Jan 4 21:32:00 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 4 Jan 2008 15:32:00 -0600 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <200801042030.26315.toad@amphibian.dyndns.org> References: <20080104182211.0332D390A59@freenetproject.org> <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> <200801042030.26315.toad@amphibian.dyndns.org> Message-ID: <497A6D42-5D34-4E3C-A493-6862EA9307A3@emu.freenetproject.org> On Jan 4, 2008, at 2:30 PM, Matthew Toseland wrote: > On Friday 04 January 2008 18:32, Robert Hailey wrote: >> >> Apparently until this revision 16886, (so long as any one node does >> not timeout) a node will take as long as necessary to exhaust >> routable >> peers. Even long after the original requestor has given up on that >> node. > > Yes. Is this bad? Obviously there are limits - if it gets a post- > accepted > timeout on any one node it will finish the request. > > Generally I think this is probably a good thing - the data is > wanted, so why > not find it? It will be cached and will be transferred later. With > ULPRs it > will even be transferred when we complete, despite the timeout. So in the best case, the timeout value was just-wrong-enough for overall network topology towards this address. In the worst case, the data does not exist (e.g. looking for a newer USK, or the next frost message)... which might actually be the common case. In either case, resuming a request after we know that the upstream peer has forgotten about it could be very bad. Assuming 20 peers (ala opennet), the theoretical worst-case-per-node is that the last new request will leave the node about 40 minutes from when it entered the node. To the best of my knowledge, all of the upstream nodes will not respond with the LOOP rejection before then. And even well before the worst case, this effect can accrue across many nodes in the path. >> I suspect good and bad from this. On the one hand, all nodes will >> become much less busy (increase throughput, decrease latency), but >> the >> data may be not be found at all (as presently the node may continue >> to >> search all its peers and it be cached for next time). > > Right, the advantage of the current system is that the data will > probably be > found eventually, and the next time the request is made it will be > available > before it times out. But if it's hard to find, if we timeout as you > suggest, > it may not be found. I think the more-correct solution to that problem is to route less to nodes which take so long, or choose them last. Which is to say, if a certain node (or path to a node) is making us timeout, avoid it; benefiting the general network health and responsiveness. > The only real reason I can think of to timeout as you do below would > be for > load balancing fairness: If you make a request, you had better be > prepared > (bandwidth wise) to accept the resulting data, if you're not, that's > a denial > of service attack. Load management relies on propagating the load > back to the > requestor. But the requestor can have only a very limited influence > on timing > out here so I don't consider it to be valid. Perhaps related to this is the furthestStoreSuccess stat, which will usually near-instantly stick to ~0.5. Sometimes exactly 0.5... which means that (wherever the request came from), my node was the absolute- farthest node possible, last on his list to call, and all my peers where closer. But weirdest of all, that the data was actually in the store. More evidence (IMHO) for store-position-bias :) Intuitively, we should be near the data the network sends us, and not search the whole network for the data we want. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080104/a700a1e2/attachment.htm From robert at emu.freenetproject.org Sat Jan 5 00:50:37 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 4 Jan 2008 18:50:37 -0600 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <497A6D42-5D34-4E3C-A493-6862EA9307A3@emu.freenetproject.org> References: <20080104182211.0332D390A59@freenetproject.org> <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> <200801042030.26315.toad@amphibian.dyndns.org> <497A6D42-5D34-4E3C-A493-6862EA9307A3@emu.freenetproject.org> Message-ID: <1A4DFD4D-06BA-4353-8C22-D3DDE8704880@emu.freenetproject.org> On Jan 4, 2008, at 2:30 PM, Matthew Toseland wrote: > On Friday 04 January 2008 18:32, Robert Hailey wrote: >> >> Apparently until this revision 16886, (so long as any one node does >> not timeout) a node will take as long as necessary to exhaust >> routable >> peers. Even long after the original requestor has given up on that >> node. > > Yes. Is this bad? Obviously there are limits - if it gets a post- > accepted > timeout on any one node it will finish the request. > > Generally I think this is probably a good thing - the data is > wanted, so why > not find it? It will be cached and will be transferred later. With > ULPRs it > will even be transferred when we complete, despite the timeout. Interestingly (now that I have got the simulator running), this 'general timeout' appears even in simulations between nodes on the same machine. Unless I coded something wrong, perhaps there is an added delay or missing response somewhere which is not obvious? -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080104/bc088a97/attachment.htm From robert at emu.freenetproject.org Sat Jan 5 04:00:07 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 4 Jan 2008 22:00:07 -0600 Subject: [freenet-dev] [freenet-cvs] r16826 - trunk/freenet/src/freenet/node In-Reply-To: <200801031511.36969.toad@amphibian.dyndns.org> References: <20071228165118.B96914798DD@freenetproject.org> <200801031511.36969.toad@amphibian.dyndns.org> Message-ID: <1BF6FCE3-C634-44A9-AA23-97D9AAE5C862@emu.freenetproject.org> On Jan 3, 2008, at 9:11 AM, Matthew Toseland wrote: > Isn't sendSync() a longer timeout than the waitFor()? We could > change that > though, pass in a timeout..? I think that this place is one of the few exceptions to the sendAsync- is-better-for-acks, and that's why I changed it back. Whereas requests are more of a 'trigger' and inserts 'pushing'... we require information from the node anyway (the actual data insert), and we *will* wait for it... either the 10minutes+DATA_INSERT_TIMEOUT, or just the DATA_INSERT_TIMEOUT. So using sendSync (to absorb the time of the send queues) really does help! Otherwise an error is propagated back to the source (thinking that they asked for an insert & forgot the data). Rather than passing in a timeout, would would help more is to somehow know if it ever made it of the queue. e.g. If it takes longer than 10 minutes, remove it from the send queue and throw a 'waited-too-long' exception. -- Robert Hailey > On Friday 28 December 2007 16:51, robert at freenetproject.org wrote: >> Author: robert >> Date: 2007-12-28 16:51:18 +0000 (Fri, 28 Dec 2007) >> New Revision: 16826 >> >> Modified: >> trunk/freenet/src/freenet/node/InsertHandler.java >> Log: >> less timeouts which are really disconnects >> >> >> Modified: trunk/freenet/src/freenet/node/InsertHandler.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/InsertHandler.java 2007-12-28 >> 16:50:28 > UTC (rev 16825) >> +++ trunk/freenet/src/freenet/node/InsertHandler.java 2007-12-28 >> 16:51:18 > UTC (rev 16826) >> @@ -91,7 +91,8 @@ >> // Send Accepted >> Message accepted = DMT.createFNPAccepted(uid); >> try { >> - source.sendAsync(accepted, null, 0, this); >> + //Using sendSync here will help the next message filter not >> timeout... > wait here or at the message filter. >> + source.sendSync(accepted, this); >> } catch (NotConnectedException e1) { >> if(logMINOR) Logger.minor(this, "Lost connection to source"); >> return; >> >> From m.rogers at cs.ucl.ac.uk Sat Jan 5 04:16:15 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Sat, 05 Jan 2008 04:16:15 +0000 Subject: [freenet-dev] Vulnerability in inserting the manifest before finishing In-Reply-To: <200801042034.34556.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801041434.20713.toad@amphibian.dyndns.org> <477E8FBE.5080209@cs.ucl.ac.uk> <200801042034.34556.toad@amphibian.dyndns.org> Message-ID: <477F048F.40808@cs.ucl.ac.uk> Matthew Toseland wrote: > Having said that, is going through untrusted nodes as gateways from one > trusted node to another a problem? As long as we can choose two cell members > to go through, we should be okay? So we onion-encrypt the request with the keys of two cell members, one of which is the exit node, and allow the request to find its own way from the initiator to the first cell member and from there to the second? Sounds plausible... Cheers, Michael From m.rogers at cs.ucl.ac.uk Sat Jan 5 05:15:44 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Sat, 05 Jan 2008 05:15:44 +0000 Subject: [freenet-dev] Tunnels vs premix routing was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <200801042039.50495.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801041432.54681.toad@amphibian.dyndns.org> <477E8A77.5090307@cs.ucl.ac.uk> <200801042039.50495.toad@amphibian.dyndns.org> Message-ID: <477F1280.40107@cs.ucl.ac.uk> Matthew Toseland wrote: > The 50% chance of decrementing > at 10 means that 3 resets equals 6 nodes with HTL 10... It's more complicated than that... if the HTL is 10 and the closest location isn't the previous hop's location, then the attacker knows the previous hop doesn't decrement at 10. Likewise if the HTL is 9 and the closest location isn't the previous hop's location, the previous hop decrements at 10. Either way, the attacker can use that knowledge for the rest of the session. > When an attacker gets a tunneled request, he knows 1) there is a 20% chance > that the node is the originator, and 2) that he is close to the originator, > because he is early on the route: the tunnel is only active at the beginning. Longer tunnels would help in both cases (at a cost, of course). Ideally the tunnels would be long enough to reach any node with equal probability, but maybe that's not realistic... Let's say we need an average of 10 hops. A weighted coin gives an exponential hop count distribution (actually geometric but close enough), so 25% of tunnels will have <= 2.9 hops, 50% will have <= 6.9 hops, 75% will have <= 13.9 hops. Likewise the attacker will know there's a 25% chance the initiator is within 2.9 hops, etc. (Not sure how to handle the rounding here but you get the picture.) > For a regular request, the HTL can be 10 much later on in the request. In > which case the predecessor sample is useless, and the key is likely close to > the location of the attacker. Good point about the distance - can the attacker give less weight to samples where the requested key is close to the attacker, on the assumption that they're more likely to have travelled several hops already? > Somewhere on this thread you said that the probability of two related tunnels > coming from the same node is independant of that node being the originator. I > don't understand that. I didn't mean that exactly - the fact that both tunnels pass through X *does* suggest that X is the initiator, I'm just saying it's irrelevant where they go after leaving X. An attacker doesn't learn any more by linking two tunnels that pass from X to different neighbours than he learns by linking two tunnels that pass from X to the same neighbour, because the paths of the tunnels are independent, so the fact that they did or didn't part company at X doesn't tell the attacker anything about where they originated. It was a minor point really. Cheers, Michael From batosai at batosai.net Sun Jan 6 11:45:44 2008 From: batosai at batosai.net (Julien Cornuwel) Date: Sun, 06 Jan 2008 12:45:44 +0100 Subject: [freenet-dev] French translation update for 1097 Message-ID: <4780BF68.2080600@batosai.net> Hi, Here is an updated translation file. I noticed that the SSL strings are all about keys. Shouldn't we talk about certificates ? Except for the private key of course. Regards -------------- next part -------------- An embedded and charset-unspecified text was scrubbed... Name: freenet.l10n.fr.override.properties Url: http://emu.freenetproject.org/pipermail/devl/attachments/20080106/721e677f/attachment.txt -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 250 bytes Desc: OpenPGP digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080106/721e677f/attachment.pgp From nextgens at freenetproject.org Sun Jan 6 19:24:12 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Sun, 6 Jan 2008 20:24:12 +0100 Subject: [freenet-dev] [freenet-cvs] r16942 - trunk/freenet/src/freenet/node In-Reply-To: <20080106181045.C821147986F@freenetproject.org> References: <20080106181045.C821147986F@freenetproject.org> Message-ID: <20080106192410.GA4515@freenetproject.org> * zothar at freenetproject.org [2008-01-06 18:10:45]: > Author: zothar > Date: 2008-01-06 18:10:45 +0000 (Sun, 06 Jan 2008) > New Revision: 16942 > > Modified: > trunk/freenet/src/freenet/node/Node.java > Log: > Refactor N2NTM dependence on the peer being a darknet peer > > Modified: trunk/freenet/src/freenet/node/Node.java > =================================================================== > --- trunk/freenet/src/freenet/node/Node.java 2008-01-06 16:53:08 UTC (rev 16941) > +++ trunk/freenet/src/freenet/node/Node.java 2008-01-06 18:10:45 UTC (rev 16942) > @@ -2529,15 +2529,18 @@ > } > > public void receivedNodeToNodeMessage(PeerNode src, int type, ShortBuffer messageData, boolean partingMessage) { > - if(!(src instanceof DarknetPeerNode)) { > - Logger.error(this, "Got N2NTM from opennet node ?!?!?!: from "+src); > - return; > + boolean fromDarknet = false; > + if(src instanceof DarknetPeerNode) { > + fromDarknet = true; > } > - DarknetPeerNode source = (DarknetPeerNode)src; > + DarknetPeerNode darkSource = (DarknetPeerNode)src; > > if(type == Node.N2N_MESSAGE_TYPE_FPROXY) { > - > - Logger.normal(this, "Received N2NM from '"+source.getPeer()+"'"); > + if(!fromDarknet) { > + Logger.error(this, "Got N2NTM from non-darknet node ?!?!?!: from "+src); > + return; > + } > + Logger.normal(this, "Received N2NTM from '"+darkSource.getPeer()+"'"); That's gonna throw a classcast exception if fromDarknet is false NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080106/3e56b601/attachment.pgp From freenet-devl at david.sowder.com Sun Jan 6 20:46:42 2008 From: freenet-devl at david.sowder.com (David Sowder (Zothar)) Date: Sun, 06 Jan 2008 14:46:42 -0600 Subject: [freenet-dev] [freenet-cvs] r16942 - trunk/freenet/src/freenet/node In-Reply-To: <20080106192410.GA4515@freenetproject.org> References: <20080106181045.C821147986F@freenetproject.org> <20080106192410.GA4515@freenetproject.org> Message-ID: <47813E32.70609@david.sowder.com> Florent Daigni?re wrote: > * zothar at freenetproject.org [2008-01-06 18:10:45]: > > >> Author: zothar >> Date: 2008-01-06 18:10:45 +0000 (Sun, 06 Jan 2008) >> New Revision: 16942 >> >> Modified: >> trunk/freenet/src/freenet/node/Node.java >> Log: >> Refactor N2NTM dependence on the peer being a darknet peer >> >> Modified: trunk/freenet/src/freenet/node/Node.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/Node.java 2008-01-06 16:53:08 UTC (rev 16941) >> +++ trunk/freenet/src/freenet/node/Node.java 2008-01-06 18:10:45 UTC (rev 16942) >> @@ -2529,15 +2529,18 @@ >> } >> >> public void receivedNodeToNodeMessage(PeerNode src, int type, ShortBuffer messageData, boolean partingMessage) { >> - if(!(src instanceof DarknetPeerNode)) { >> - Logger.error(this, "Got N2NTM from opennet node ?!?!?!: from "+src); >> - return; >> + boolean fromDarknet = false; >> + if(src instanceof DarknetPeerNode) { >> + fromDarknet = true; >> } >> - DarknetPeerNode source = (DarknetPeerNode)src; >> + DarknetPeerNode darkSource = (DarknetPeerNode)src; >> >> if(type == Node.N2N_MESSAGE_TYPE_FPROXY) { >> - >> - Logger.normal(this, "Received N2NM from '"+source.getPeer()+"'"); >> + if(!fromDarknet) { >> + Logger.error(this, "Got N2NTM from non-darknet node ?!?!?!: from "+src); >> + return; >> + } >> + Logger.normal(this, "Received N2NTM from '"+darkSource.getPeer()+"'"); >> > > That's gonna throw a classcast exception if fromDarknet is false > I forgot the needed if. It should be fixed now. (I forget the revision #) From robert at emu.freenetproject.org Mon Jan 7 18:25:59 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Mon, 7 Jan 2008 12:25:59 -0600 Subject: [freenet-dev] current trunk In-Reply-To: <20080106195725.GB4515@freenetproject.org> References: <20080106195725.GB4515@freenetproject.org> Message-ID: On Jan 6, 2008, at 1:57 PM, Florent Daigni?re wrote: > Hey! > > I'm currently running latest trunk and have noticed that the > unclaimedFIFOSize has grown lately... Didn't you introduce a bug > in the message matching somehow ? :) > > NextGen$ r16905, yes... it is exactly a bug in the message matching system :) It is so weird though, that in local simulations +90% of requests that were caught waiting for a reply which never came received exactly one FNPRejectOverload (and through that path). Reverted in r16956. -- Robert Hailey From martin.nyhus at sensewave.com Mon Jan 7 18:47:24 2008 From: martin.nyhus at sensewave.com (Martin Nyhus) Date: Mon, 7 Jan 2008 19:47:24 +0100 Subject: [freenet-dev] [freenet-cvs] r16923 - trunk/freenet/src/freenet/store In-Reply-To: <20080105212637.A756747A2CB@freenetproject.org> References: <20080105212637.A756747A2CB@freenetproject.org> Message-ID: <200801071947.31610.martin.nyhus@sensewave.com> On Saturday 5. January 2008 22:26:37 toad at freenetproject.org wrote: > Author: toad > Date: 2008-01-05 21:26:37 +0000 (Sat, 05 Jan 2008) > New Revision: 16923 > > Modified: > trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java > Log: > demote to DEBUG level logging [...] > @@ -1580,9 +1573,9 @@ > if(keysRAF != null) { > keysRAF.seek(blockNum * keyLength); > keysRAF.write(fullKey); > - if(logMINOR) > + if(logDEBUG) > Logger.minor(this, "Written full key length "+fullKey.length+" to block "+blockNum+" at "+(blockNum * keyLength)); > - } else if(logMINOR && storeType == TYPE_SSK) { > + } else if(logDEBUG && storeType == TYPE_SSK) { > Logger.minor(this, "Not writing full key length "+fullKey.length+" for block "+blockNum); > } Shouldn't Logger.minor be Logger.debug here? -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: This is a digitally signed message part. Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080107/4cb5b958/attachment.pgp From toad at amphibian.dyndns.org Tue Jan 8 21:33:50 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Tue, 8 Jan 2008 21:33:50 +0000 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <1A4DFD4D-06BA-4353-8C22-D3DDE8704880@emu.freenetproject.org> References: <20080104182211.0332D390A59@freenetproject.org> <497A6D42-5D34-4E3C-A493-6862EA9307A3@emu.freenetproject.org> <1A4DFD4D-06BA-4353-8C22-D3DDE8704880@emu.freenetproject.org> Message-ID: <200801082133.51052.toad@amphibian.dyndns.org> On Saturday 05 January 2008 00:50, Robert Hailey wrote: > > On Jan 4, 2008, at 2:30 PM, Matthew Toseland wrote: > > > On Friday 04 January 2008 18:32, Robert Hailey wrote: > >> > >> Apparently until this revision 16886, (so long as any one node does > >> not timeout) a node will take as long as necessary to exhaust > >> routable > >> peers. Even long after the original requestor has given up on that > >> node. > > > > Yes. Is this bad? Obviously there are limits - if it gets a post- > > accepted > > timeout on any one node it will finish the request. > > > > Generally I think this is probably a good thing - the data is > > wanted, so why > > not find it? It will be cached and will be transferred later. With > > ULPRs it > > will even be transferred when we complete, despite the timeout. > > Interestingly (now that I have got the simulator running), this > 'general timeout' appears even in simulations between nodes on the > same machine. Unless I coded something wrong, perhaps there is an > added delay or missing response somewhere which is not obvious? Entirely possible. Fixing it would be better than an arbitrary cutoff when we are still able to potentially find the data, and still have enough HTL to do so. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080108/1b50a9c7/attachment.pgp From toad at amphibian.dyndns.org Tue Jan 8 21:37:34 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Tue, 8 Jan 2008 21:37:34 +0000 Subject: [freenet-dev] [freenet-cvs] r16885 - trunk/freenet/src/freenet/node/useralerts In-Reply-To: <20080104172235.08C383909DA@freenetproject.org> References: <20080104172235.08C383909DA@freenetproject.org> Message-ID: <200801082137.34540.toad@amphibian.dyndns.org> On Friday 04 January 2008 17:22, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-04 17:22:34 +0000 (Fri, 04 Jan 2008) > New Revision: 16885 > > Modified: > trunk/freenet/src/freenet/node/useralerts/UserAlertManager.java > Log: > more future-proof and support less-than-minor messages > I'm not sure I get it - it's more future proof because we don't need to put a messageTypes++ in if we add another type of message? > > Modified: trunk/freenet/src/freenet/node/useralerts/UserAlertManager.java > =================================================================== > --- trunk/freenet/src/freenet/node/useralerts/UserAlertManager.java 2008-01-04 17:07:15 UTC (rev 16884) > +++ trunk/freenet/src/freenet/node/useralerts/UserAlertManager.java 2008-01-04 17:22:34 UTC (rev 16885) > @@ -204,8 +204,9 @@ > separatorNeeded = true; > messageTypes++; > } > - if (messageTypes > 1) { > - alertSummaryString.append(" | "); > + if (messageTypes != 1) { > + if (separatorNeeded) > + alertSummaryString.append(" | "); > alertSummaryString.append(l10n("totalLabel")).append(' ').append(totalNumber); > } > HTMLNode summaryBox = null; > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080108/7c5fab5f/attachment.pgp From toad at amphibian.dyndns.org Tue Jan 8 20:39:37 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Tue, 8 Jan 2008 20:39:37 +0000 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <497A6D42-5D34-4E3C-A493-6862EA9307A3@emu.freenetproject.org> References: <20080104182211.0332D390A59@freenetproject.org> <200801042030.26315.toad@amphibian.dyndns.org> <497A6D42-5D34-4E3C-A493-6862EA9307A3@emu.freenetproject.org> Message-ID: <200801082039.43208.toad@amphibian.dyndns.org> On Friday 04 January 2008 21:32, Robert Hailey wrote: > > On Jan 4, 2008, at 2:30 PM, Matthew Toseland wrote: > > > On Friday 04 January 2008 18:32, Robert Hailey wrote: > >> > >> Apparently until this revision 16886, (so long as any one node does > >> not timeout) a node will take as long as necessary to exhaust > >> routable > >> peers. Even long after the original requestor has given up on that > >> node. > > > > Yes. Is this bad? Obviously there are limits - if it gets a post- > > accepted > > timeout on any one node it will finish the request. > > > > Generally I think this is probably a good thing - the data is > > wanted, so why > > not find it? It will be cached and will be transferred later. With > > ULPRs it > > will even be transferred when we complete, despite the timeout. > > So in the best case, the timeout value was just-wrong-enough for > overall network topology towards this address. In the worst case, the > data does not exist (e.g. looking for a newer USK, or the next frost > message)... which might actually be the common case. In which case, it will run out of HTL reasonably quickly. I don't see a problem. > > In either case, resuming a request after we know that the upstream > peer has forgotten about it could be very bad. Assuming 20 peers (ala > opennet), the theoretical worst-case-per-node is that the last new > request will leave the node about 40 minutes from when it entered the > node. Not possible because of HTL. We will DNF (or timeout) before we've visited all 20 nodes. We don't allow RNFs to increase the HTL or change the nearest-loc-found, but we do allow them to decrease the HTL. And we decrement the HTL unless nearestLoc improves (the HTL decrement/reset code really needs to be looked at, it's far from clear and may be wrong in places). One caveat: at HTL=10 or HTL=1, whether or not the node decrements is determined probabilistically (the decision is made once per source node). > To the best of my knowledge, all of the upstream nodes will not > respond with the LOOP rejection before then. And even well before the > worst case, this effect can accrue across many nodes in the path. If the same request is routed to a node which is already running it, it rejects it with RejectedLoop. If it's routed to a node which has recently ran it, it again rejects it. If it is a different request for the same key, it may be coalesced. > > >> I suspect good and bad from this. On the one hand, all nodes will > >> become much less busy (increase throughput, decrease latency), but > >> the > >> data may be not be found at all (as presently the node may continue > >> to > >> search all its peers and it be cached for next time). > > > > Right, the advantage of the current system is that the data will > > probably be > > found eventually, and the next time the request is made it will be > > available > > before it times out. But if it's hard to find, if we timeout as you > > suggest, > > it may not be found. > > I think the more-correct solution to that problem is to route less to > nodes which take so long, or choose them last. Which is to say, if a > certain node (or path to a node) is making us timeout, avoid it; > benefiting the general network health and responsiveness. Maybe. Maybe the key is in the wrong place on the network, the topology is perverse, but the request does eventually return the data. In which case it gets cached and propagated. > > > The only real reason I can think of to timeout as you do below would > > be for > > load balancing fairness: If you make a request, you had better be > > prepared > > (bandwidth wise) to accept the resulting data, if you're not, that's > > a denial > > of service attack. Load management relies on propagating the load > > back to the > > requestor. But the requestor can have only a very limited influence > > on timing > > out here so I don't consider it to be valid. > > Perhaps related to this is the furthestStoreSuccess stat, which will > usually near-instantly stick to ~0.5. Sometimes exactly 0.5... which > means that (wherever the request came from), my node was the absolute- > farthest node possible, last on his list to call, and all my peers > where closer. But weirdest of all, that the data was actually in the > store. More evidence (IMHO) for store-position-bias :) Intuitively, we > should be near the data the network sends us, and not search the whole > network for the data we want. More evidence that we have too much location churn, presumably caused by connection churn, node churn, and old perverse topologies. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080108/d1e5a540/attachment.pgp From toad at amphibian.dyndns.org Tue Jan 8 20:27:27 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Tue, 8 Jan 2008 20:27:27 +0000 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> References: <20080104182211.0332D390A59@freenetproject.org> <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> Message-ID: <200801082027.32705.toad@amphibian.dyndns.org> On Friday 04 January 2008 18:32, Robert Hailey wrote: > > Apparently until this revision 16886, (so long as any one node does > not timeout) a node will take as long as necessary to exhaust routable > peers. Even long after the original requestor has given up on that node. Is there any evidence that this happens in practice? Surely the HTL should prevent excessive searching in most cases? -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080108/330aa9ed/attachment.pgp From toad at amphibian.dyndns.org Tue Jan 8 19:16:03 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Tue, 8 Jan 2008 19:16:03 +0000 Subject: [freenet-dev] [freenet-cvs] r16923 - trunk/freenet/src/freenet/store In-Reply-To: <200801071947.31610.martin.nyhus@sensewave.com> References: <20080105212637.A756747A2CB@freenetproject.org> <200801071947.31610.martin.nyhus@sensewave.com> Message-ID: <200801081916.08471.toad@amphibian.dyndns.org> On Monday 07 January 2008 18:47, Martin Nyhus wrote: > On Saturday 5. January 2008 22:26:37 toad at freenetproject.org wrote: > > Author: toad > > Date: 2008-01-05 21:26:37 +0000 (Sat, 05 Jan 2008) > > New Revision: 16923 > > > > Modified: > > trunk/freenet/src/freenet/store/BerkeleyDBFreenetStore.java > > Log: > > demote to DEBUG level logging > [...] > > @@ -1580,9 +1573,9 @@ > > if(keysRAF != null) { > > keysRAF.seek(blockNum * keyLength); > > keysRAF.write(fullKey); > > - if(logMINOR) > > + if(logDEBUG) > > Logger.minor(this, "Written full key length "+fullKey.length+" to > block "+blockNum+" at "+(blockNum * keyLength)); > > - } else if(logMINOR && storeType == TYPE_SSK) { > > + } else if(logDEBUG && storeType == TYPE_SSK) { > > Logger.minor(this, "Not writing full key length "+fullKey.length+" for > block "+blockNum); > > } > > Shouldn't Logger.minor be Logger.debug here? Indeed it should. Will fix. > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080108/28714e10/attachment.pgp From toad at amphibian.dyndns.org Tue Jan 8 20:27:27 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Tue, 8 Jan 2008 20:27:27 +0000 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> References: <20080104182211.0332D390A59@freenetproject.org> <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> Message-ID: <200801082027.32705.toad@amphibian.dyndns.org> On Friday 04 January 2008 18:32, Robert Hailey wrote: > > Apparently until this revision 16886, (so long as any one node does > not timeout) a node will take as long as necessary to exhaust routable > peers. Even long after the original requestor has given up on that node. Is there any evidence that this happens in practice? Surely the HTL should prevent excessive searching in most cases? -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080108/330aa9ed/attachment-0001.pgp From robert at emu.freenetproject.org Tue Jan 8 22:57:21 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Tue, 8 Jan 2008 16:57:21 -0600 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <200801082133.51052.toad@amphibian.dyndns.org> References: <20080104182211.0332D390A59@freenetproject.org> <497A6D42-5D34-4E3C-A493-6862EA9307A3@emu.freenetproject.org> <1A4DFD4D-06BA-4353-8C22-D3DDE8704880@emu.freenetproject.org> <200801082133.51052.toad@amphibian.dyndns.org> Message-ID: <690CA447-2388-449B-8030-BE7F56BF6C6D@emu.freenetproject.org> On Jan 8, 2008, at 3:33 PM, Matthew Toseland wrote: > On Saturday 05 January 2008 00:50, Robert Hailey wrote: >> >> Interestingly (now that I have got the simulator running), this >> 'general timeout' appears even in simulations between nodes on the >> same machine. Unless I coded something wrong, perhaps there is an >> added delay or missing response somewhere which is not obvious? > > Entirely possible. Fixing it would be better than an arbitrary > cutoff when we > are still able to potentially find the data, and still have enough > HTL to do > so. On Jan 8, 2008, at 2:27 PM, Matthew Toseland wrote: > On Friday 04 January 2008 18:32, Robert Hailey wrote: >> >> Apparently until this revision 16886, (so long as any one node does >> not timeout) a node will take as long as necessary to exhaust >> routable >> peers. Even long after the original requestor has given up on that >> node. > > Is there any evidence that this happens in practice? Surely the HTL > should > prevent excessive searching in most cases? There is, in fact. The timeout itself (which I have been running on my node for a while) is evidence of the behavior (which to be seems incorrect). Jan 08, 2008 20:03:41:146 (freenet.node.RequestSender, RequestSender for UID -3998139406700477577, ERROR): discontinuing non-local request search, general timeout (6 attempts, 3 overloads) ... Jan 08, 2008 20:12:21:226 (freenet.node.RequestSender, RequestSender for UID 60170596711015291, ERROR): discontinuing non-local request search, general timeout (1 attempts, 3 overloads) You see... in the first log statement the node tried six peers before running out of time. In the second case (which occurs quite frequently), the node used the entire 2 minutes waiting on a response from one node (FETCH_TIMEOUT); if it were allowed to continue to the next node it could (65%) spend another 2 minutes on just-that-node. >> To the best of my knowledge, all of the upstream nodes will not >> respond with the LOOP rejection before then. And even well before the >> worst case, this effect can accrue across many nodes in the path. > > If the same request is routed to a node which is already running it, > it > rejects it with RejectedLoop. If it's routed to a node which has > recently ran > it, it again rejects it. If it is a different request for the same > key, it > may be coalesced. If you mean that the RECENTLY_FAILED mechanism would keep this in check... I see this idea in many places, but I cannot see where this is actually implemented. The only place I see that makes a FNPRecentlyFailed message is in RequestHandler (upon it's RequestSender having received one). Presently the node will "RejectLoop" if it is one of the last 10000 completed requests. My node runs through that many requests in about 16 minutes. With this logging statement it is already shown that a request can last longer than 2 minutes for one peer (and most nodes have 20). If you assume that a request takes 4 minutes (two peers, VERY optimistic), then it would then only take 4 nodes ('near' each other) to generate a request-live-lock (absent the HTL; the request would never drop from the network); each trying two of it's other peers, and then the next in the 4-chain. I do not think that this timeout I added is arbitrary. As I understand Ian's original networking theory, a request is not valid after the originator has timed out. In much the same way that a single node fatally timing out collapses the request chain, so too should a node 'taking too long' (as that node *IS* the one fatally timing out the chain). But on the other hand, I do understand your point about the HTL, and that it would keep the request from continuing indefinitely; it seems like it could be quite a waste of network resources. Certainly beyond that point in time (where the requester has fatally timed out) no response should be sent back to the source (that could be many of the unclaimed fifo packets); or maybe just if the data is finally found (you mentioned ULPRs). -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080108/8ea682c3/attachment.htm From robert at emu.freenetproject.org Wed Jan 9 17:14:54 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Wed, 9 Jan 2008 11:14:54 -0600 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <200801082027.32705.toad@amphibian.dyndns.org> References: <20080104182211.0332D390A59@freenetproject.org> <91204D41-B734-4D7B-8D14-81CD7A03C525@emu.freenetproject.org> <200801082027.32705.toad@amphibian.dyndns.org> Message-ID: <0CC1E253-3C9F-483A-8137-D43B74F2E86B@emu.freenetproject.org> On Jan 8, 2008, at 2:27 PM, Matthew Toseland wrote: > On Friday 04 January 2008 18:32, Robert Hailey wrote: >> >> Apparently until this revision 16886, (so long as any one node does >> not timeout) a node will take as long as necessary to exhaust >> routable >> peers. Even long after the original requestor has given up on that >> node. > > Is there any evidence that this happens in practice? Surely the HTL > should > prevent excessive searching in most cases? I have reverted r16886, as it appears to be based on a misunderstand of how requests work against the topology of the network (r16980). Instead of canceling the search request, I have put similar logic in RequestHandler to simply not respond to very old requests (r16982), and not to hang onto the thread if no response is therefore necessary (r16983). r16983, (given the timeout) makes extra calls to rejectedOverload(); I imagine that at the timeout (every 2 minutes while the local request is running) they are benign, but I can code up an infinite wait (as before) if needed. I'm going to try and stay away from high level changes for a while, but at your suggestion I will look at the HTL code. -- Robert Hailey From robert at emu.freenetproject.org Wed Jan 9 19:59:29 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Wed, 9 Jan 2008 13:59:29 -0600 Subject: [freenet-dev] htl problem was r16886 In-Reply-To: <200801082039.43208.toad@amphibian.dyndns.org> References: <20080104182211.0332D390A59@freenetproject.org> <200801042030.26315.toad@amphibian.dyndns.org> <497A6D42-5D34-4E3C-A493-6862EA9307A3@emu.freenetproject.org> <200801082039.43208.toad@amphibian.dyndns.org> Message-ID: On Jan 8, 2008, at 2:39 PM, Matthew Toseland wrote: >> In either case, resuming a request after we know that the upstream >> peer has forgotten about it could be very bad. Assuming 20 peers (ala >> opennet), the theoretical worst-case-per-node is that the last new >> request will leave the node about 40 minutes from when it entered the >> node. > > Not possible because of HTL. We will DNF (or timeout) before we've > visited all > 20 nodes. We don't allow RNFs to increase the HTL or change the > nearest-loc-found, but we do allow them to decrease the HTL. And we > decrement > the HTL unless nearestLoc improves (the HTL decrement/reset code > really needs > to be looked at, it's far from clear and may be wrong in places). > One caveat: > at HTL=10 or HTL=1, whether or not the node decrements is determined > probabilistically (the decision is made once per source node). It looks like the htl is decremented with respect to the source of the node (not the node we are routing to), so at 10 or 1 if the request came from a node w/o the probabalitic decrement (or if prob. decrement is disabled!) then the standard request search will try all it's peers. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080109/9cc417c9/attachment.htm From jargonautti at hotmail.com Thu Jan 10 09:55:09 2008 From: jargonautti at hotmail.com (Jusa Saari) Date: Thu, 10 Jan 2008 11:55:09 +0200 Subject: [freenet-dev] Unkown size in FProxy global queue? Message-ID: FProxy global queue page often shows that a nonzero percentage of a file has been downloaded, yet claims that the file size is "unkown". How is this possible? Surely the total size must be known to calculate the pecentage? From Volodya at WhenGendarmeSleeps.org Thu Jan 10 18:46:08 2008 From: Volodya at WhenGendarmeSleeps.org (Volodya) Date: Thu, 10 Jan 2008 21:46:08 +0300 Subject: [freenet-dev] Unkown size in FProxy global queue? In-Reply-To: References: Message-ID: <478667F0.60503@WhenGendarmeSleeps.org> Jusa Saari ?????: > FProxy global queue page often shows that a nonzero percentage of a file > has been downloaded, yet claims that the file size is "unkown". How is > this possible? Surely the total size must be known to calculate the > pecentage? It is possible that what you see is the percentage of all known blocks, some blocks might not yet be known since not all the metadata is yet downloaded. Unlike in the old versions of freenet metadata can be broken up into pieces. - Volodya -- http://freedom.libsyn.com/ Voice of Freedom, Radical Podcast http://eng.anarchopedia.org/ Anarchopedia, A Free Knowledge Portal "None of us are free until all of us are free." ~ Mihail Bakunin From toad at amphibian.dyndns.org Thu Jan 10 21:48:09 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 10 Jan 2008 21:48:09 +0000 Subject: [freenet-dev] [freenet-cvs] r16898 - trunk/freenet/src/freenet/node/fcp In-Reply-To: <20080104233420.D21C43920F8@freenetproject.org> References: <20080104233420.D21C43920F8@freenetproject.org> Message-ID: <200801102148.09337.toad@amphibian.dyndns.org> On Friday 04 January 2008 23:34, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-04 23:34:20 +0000 (Fri, 04 Jan 2008) > New Revision: 16898 > > Modified: > trunk/freenet/src/freenet/node/fcp/FCPServer.java > Log: > don't get network port if disabled, lazy port-binding Does this break configuring such things while disabled? I.e. are the configs registered and functional even when the service is disabled? > > > Modified: trunk/freenet/src/freenet/node/fcp/FCPServer.java > =================================================================== > --- trunk/freenet/src/freenet/node/fcp/FCPServer.java 2008-01-04 23:00:50 UTC (rev 16897) > +++ trunk/freenet/src/freenet/node/fcp/FCPServer.java 2008-01-04 23:34:20 UTC (rev 16898) > @@ -64,6 +64,7 @@ > private static boolean ssl = false; > public final boolean enabled; > String bindTo; > + private String allowedHosts; > AllowedHosts allowedHostsFullAccess; > final WeakHashMap clientsByName; > final FCPClient globalClient; > @@ -98,6 +99,7 @@ > > public FCPServer(String ipToBindTo, String allowedHosts, String allowedHostsFullAccess, int port, Node node, NodeClientCore core, boolean persistentDownloadsEnabled, String persistentDownloadsDir, long persistenceInterval, boolean isEnabled, boolean assumeDDADownloadAllowed, boolean assumeDDAUploadAllowed) throws IOException, InvalidConfigValueException { > this.bindTo = ipToBindTo; > + this.allowedHosts=allowedHosts; > this.allowedHostsFullAccess = new AllowedHosts(allowedHostsFullAccess); > this.persistenceInterval = persistenceInterval; > this.port = port; > @@ -116,9 +118,14 @@ > defaultFetchContext = client.getFetchContext(); > defaultInsertContext = client.getInsertContext(false); > > - > globalClient = new FCPClient("Global Queue", this, null, true); > > + logMINOR = Logger.shouldLog(Logger.MINOR, this); > + } > + > + private void maybeGetNetworkInterface() { > + if (this.networkInterface!=null) return; > + > NetworkInterface tempNetworkInterface = null; > try { > if(ssl) { > @@ -133,11 +140,11 @@ > > this.networkInterface = tempNetworkInterface; > > - logMINOR = Logger.shouldLog(Logger.MINOR, this); > } > > public void maybeStart() { > if (this.enabled) { > + maybeGetNetworkInterface(); > > if(enablePersistentDownloads) { > loadPersistentRequests(); > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080110/170e3a98/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 10 21:49:39 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 10 Jan 2008 21:49:39 +0000 Subject: [freenet-dev] Unkown size in FProxy global queue? In-Reply-To: <478667F0.60503@WhenGendarmeSleeps.org> References: <478667F0.60503@WhenGendarmeSleeps.org> Message-ID: <200801102149.39391.toad@amphibian.dyndns.org> On Thursday 10 January 2008 18:46, Volodya wrote: > Jusa Saari ?????: > > FProxy global queue page often shows that a nonzero percentage of a file > > has been downloaded, yet claims that the file size is "unkown". How is > > this possible? Surely the total size must be known to calculate the > > pecentage? > > It is possible that what you see is the percentage of all known blocks, some > blocks might not yet be known since not all the metadata is yet downloaded. > Unlike in the old versions of freenet metadata can be broken up into pieces. It is a bug: one part of the node does indeed know the final size (usually), but it doesn't tell the other part. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080110/22a3b615/attachment.pgp From robert at emu.freenetproject.org Thu Jan 10 21:55:05 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Thu, 10 Jan 2008 15:55:05 -0600 Subject: [freenet-dev] [freenet-cvs] r16898 - trunk/freenet/src/freenet/node/fcp In-Reply-To: <200801102148.09337.toad@amphibian.dyndns.org> References: <20080104233420.D21C43920F8@freenetproject.org> <200801102148.09337.toad@amphibian.dyndns.org> Message-ID: <6369E312-C72D-492A-A028-FA3CEFB29F9F@emu.freenetproject.org> On Jan 10, 2008, at 3:48 PM, Matthew Toseland wrote: > On Friday 04 January 2008 23:34, robert at freenetproject.org wrote: >> Author: robert >> Date: 2008-01-04 23:34:20 +0000 (Fri, 04 Jan 2008) >> New Revision: 16898 >> >> Modified: >> trunk/freenet/src/freenet/node/fcp/FCPServer.java >> Log: >> don't get network port if disabled, lazy port-binding > > Does this break configuring such things while disabled? I.e. are the > configs > registered and functional even when the service is disabled? It should not, although I know of no way to activate the FCP/Fproxy while it is running except through Fproxy. The only exception being if the port *would-have-been-available* for binding at launch time, but is not available when activated. This interfered with the simulator, as the second node created would try and bind to the port and fail. Now, r16899 does break enabling-while-disabled, but I believe that I corrected that in r16900. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080110/2d5f162c/attachment.htm From toad at amphibian.dyndns.org Thu Jan 10 22:56:42 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 10 Jan 2008 22:56:42 +0000 Subject: [freenet-dev] [freenet-cvs] r16954 - trunk/freenet/src/freenet/node In-Reply-To: <20080106225149.A94CB3C0CE8@freenetproject.org> References: <20080106225149.A94CB3C0CE8@freenetproject.org> Message-ID: <200801102256.56037.toad@amphibian.dyndns.org> We should really define a range which can be used by the node, so that numbers above that can be used by applications when we implement FCP layer use of this mechanism (which IMHO is long overdue and would open up some interesting darknet collaboration possibilities e.g. send bookmark to peer, send thaw index to peer etc). On Sunday 06 January 2008 22:51, zothar at freenetproject.org wrote: > Author: zothar > Date: 2008-01-06 22:51:49 +0000 (Sun, 06 Jan 2008) > New Revision: 16954 > > Modified: > trunk/freenet/src/freenet/node/Node.java > trunk/freenet/src/freenet/node/PeerNode.java > Log: > Implement processing of received differential node references. It may be a bit before I get sending implemented. > > Modified: trunk/freenet/src/freenet/node/Node.java > =================================================================== > --- trunk/freenet/src/freenet/node/Node.java 2008-01-06 22:09:51 UTC (rev 16953) > +++ trunk/freenet/src/freenet/node/Node.java 2008-01-06 22:51:49 UTC (rev 16954) > @@ -2576,8 +2576,20 @@ > throw new Error(e); > } > } else if(type == Node.N2N_MESSAGE_TYPE_DIFFNODEREF) { > - // FIXME: Not yet implemented > Logger.normal(this, "Received differential node reference node to node message from "+src.getPeer()); > + SimpleFieldSet fs = null; > + try { > + fs = new SimpleFieldSet(new String(messageData.getData(), "UTF-8"), false, true); > + } catch (IOException e) { > + Logger.error(this, "IOException while parsing node to node message data", e); > + return; > + } > + try { > + src.processDiffNoderef(fs); > + } catch (FSParseException e) { > + Logger.error(this, "FSParseException while parsing node to node message data", e); > + return; > + } > } else { > Logger.error(this, "Received unknown node to node message type '"+type+"' from "+src.getPeer()); > } > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-06 22:09:51 UTC (rev 16953) > +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-06 22:51:49 UTC (rev 16954) > @@ -1934,6 +1934,14 @@ > } > > /** > + * Process a differential node reference > + * The identity must not change, or we throw. > + */ > + public void processDiffNoderef(SimpleFieldSet fs) throws FSParseException { > + processNewNoderef(fs, false, true); > + } > + > + /** > * Process a new nodereference, in compressed form. > * The identity must not change, or we throw. > */ > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080110/043f004b/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 10 23:07:58 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 10 Jan 2008 23:07:58 +0000 Subject: [freenet-dev] [freenet-cvs] r16959 - trunk/freenet/src/freenet/node In-Reply-To: <20080107200730.C0B203A13AC@freenetproject.org> References: <20080107200730.C0B203A13AC@freenetproject.org> Message-ID: <200801102308.03732.toad@amphibian.dyndns.org> Interesting! There are other ways to address the problem: - Message priorities. Why should data transfer prevent Accepted being sent? Especially as we can often combine the two. - Increasing the ACCEPTED_TIMEOUT. Why is this better? Your comment: //using conditionalSend this way might actually approximate Q-routing load balancing accross the network. Is this a good thing? We can always send the request and then reject it - and if one ubernode is transferring fast enough to keep up, we'll send all our requests to it. Why such a short SEND_TIMEOUT? On Monday 07 January 2008 20:07, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-07 20:07:30 +0000 (Mon, 07 Jan 2008) > New Revision: 16959 > > Modified: > trunk/freenet/src/freenet/node/MessageItem.java > trunk/freenet/src/freenet/node/PeerNode.java > trunk/freenet/src/freenet/node/RequestSender.java > Log: > implement conditionalSend: sendSync-with-timeout, aborts message send > > > Modified: trunk/freenet/src/freenet/node/MessageItem.java > =================================================================== > --- trunk/freenet/src/freenet/node/MessageItem.java 2008-01-07 19:31:24 UTC (rev 16958) > +++ trunk/freenet/src/freenet/node/MessageItem.java 2008-01-07 20:07:30 UTC (rev 16959) > @@ -72,4 +72,8 @@ > } > } > } > + > + public boolean isForMessage(Message msg) { > + return this.msg.equals(msg); > + } > } > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-07 19:31:24 UTC (rev 16958) > +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-07 20:07:30 UTC (rev 16959) > @@ -16,6 +16,7 @@ > import java.util.Hashtable; > import java.util.Iterator; > import java.util.LinkedList; > +import java.util.ListIterator; > import java.util.Vector; > import java.util.zip.DataFormatException; > import java.util.zip.Inflater; > @@ -968,6 +969,21 @@ > // It will wake up before the maximum coalescing delay (100ms) because > // it wakes up every 100ms *anyway*. > } > + > + private boolean maybeRemoveMessageFromQueue(Message removeMe) { > + Logger.normal(this, "attempting to remove message from send-queue: "+removeMe); > + synchronized (messagesToSendNow) { > + ListIterator i=messagesToSendNow.listIterator(); > + while (i.hasNext()) { > + MessageItem it=(MessageItem)i.next(); > + if (it.isForMessage(removeMe)) { > + i.remove(); > + return true; > + } > + } > + } > + return false; > + } > > public long getMessageQueueLengthBytes() { > long x = 0; > @@ -1396,6 +1412,36 @@ > } > > /** > + * Conceptually, send a message to this node IF it can be done within 'timeout', returing true > + * only after the message was sent (similiar to sendSync), and false if the message cannot be > + * sent to the node in that time period. As an optimization, however, this function may return > + * immediately if it is determined that the message would not leave the node within the timeout > + * period. > + */ > + public boolean conditionalSend(Message req, ByteCounter ctr, long timeout) throws NotConnectedException { > + if (timeout<=0) > + return false; > + if (getMessageQueueLengthBytes()/(getThrottle().getBandwidth()+1.0) > timeout) { > + Logger.normal(this, "conditionalSend; pre-emptively not sending message ("+timeout+"ms): "+req); > + return false; > + } > + SyncMessageCallback cb = new SyncMessageCallback(); > + sendAsync(req, cb, 0, ctr); > + cb.waitForSend(timeout); > + if (cb.done) { > + return true; > + } else { > + //best-effort: remove the message from the send queue it is ok if we can't prematurely > + //remove the item (i.e. race condition / now it is sent), but it will generate unclaimed messages, etc. > + if (!maybeRemoveMessageFromQueue(req)) > + Logger.error(this, "unable to stop transmition of request: "+req); > + else > + Logger.normal(this, "removed request from queue for timeout: "+req); > + return false; > + } > + } > + > + /** > * Enqueue a message to be sent to this node and wait up to a minute for it to be transmitted. > */ > public void sendSync(Message req, ByteCounter ctr) throws NotConnectedException { > @@ -1426,7 +1472,7 @@ > } > } > if(isConnected()) > - Logger.error(this, "Waited too long for a blocking send on " + this + " for " + PeerNode.this, new Exception("error")); > + Logger.normal(this, "Waited too long for a blocking send on " + this + " for " + PeerNode.this, new Exception("error")); > } > > public void acknowledged() { > > Modified: trunk/freenet/src/freenet/node/RequestSender.java > =================================================================== > --- trunk/freenet/src/freenet/node/RequestSender.java 2008-01-07 19:31:24 UTC (rev 16958) > +++ trunk/freenet/src/freenet/node/RequestSender.java 2008-01-07 20:07:30 UTC (rev 16959) > @@ -4,6 +4,7 @@ > package freenet.node; > > import java.util.HashSet; > +import java.util.ArrayList; > > import freenet.crypt.CryptFormatException; > import freenet.crypt.DSAPublicKey; > @@ -46,6 +47,8 @@ > public final class RequestSender implements Runnable, ByteCounter { > > // Constants > + //SEND_TIMEOUT is not a hard timeout, shoot low for low latency (250-500ms?). > + static final int SEND_TIMEOUT = 1000; > static final int ACCEPTED_TIMEOUT = 5000; > static final int FETCH_TIMEOUT = 120000; > /** Wait up to this long to get a path folding reply */ > @@ -141,6 +144,7 @@ > int rejectOverloads=0; > HashSet nodesRoutedTo = new HashSet(); > HashSet nodesNotIgnored = new HashSet(); > + ArrayList busyPeers = new ArrayList(); > while(true) { > if(logMINOR) Logger.minor(this, "htl="+htl); > if(htl == 0) { > @@ -159,9 +163,24 @@ > routeAttempts++; > > // Route it > + long sendTimeout = SEND_TIMEOUT; > + boolean usingBusyPeer=false; > PeerNode next; > next = node.peers.closerPeer(source, nodesRoutedTo, nodesNotIgnored, target, true, node.isAdvancedModeEnabled(), -1, null); > > + if (next == null && !busyPeers.isEmpty()) { > + next = (PeerNode)busyPeers.remove(0); > + usingBusyPeer=true; > + if (logMINOR) Logger.minor(this, "trying previously-found busy peer: "+next); > + //NOTE: if we are at this point, it is already presumed that the message cannot even make it off the node to this peer in SEND_TIMEOUT, use all the timeout we have left. > + sendTimeout = FETCH_TIMEOUT-(System.currentTimeMillis()-startTime); > + //Edge case, local request & we are running w/o any time left. > + if (sendTimeout < SEND_TIMEOUT && source==null) { > + if (logMINOR) Logger.minor(this, "increasing timeout for local request"); > + sendTimeout = 2*SEND_TIMEOUT; > + } > + } > + > if(next == null) { > if (logMINOR && rejectOverloads>0) > Logger.minor(this, "no more peers, but overloads ("+rejectOverloads+"/"+routeAttempts+" overloaded)"); > @@ -187,11 +206,18 @@ > // So take it from when we first started to try to send the request. > // See comments below when handling FNPRecentlyFailed for why we need this. > long timeSentRequest = System.currentTimeMillis(); > - > + > try { > //This is the first contact to this node > //async is preferred, but makes ACCEPTED_TIMEOUT much more likely for long send queues. > - next.sendAsync(req, null, 0, this); > + //using conditionalSend this way might actually approximate Q-routing load balancing accross the network. > + if (!next.conditionalSend(req, this, sendTimeout)) { > + if (usingBusyPeer) > + continue; > + Logger.normal(this, "will try this peer later if no others are available"); > + busyPeers.add(next); > + continue; > + } > } catch (NotConnectedException e) { > Logger.minor(this, "Not connected"); > continue; > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080110/13c25cae/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 10 23:10:09 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 10 Jan 2008 23:10:09 +0000 Subject: [freenet-dev] [freenet-cvs] r16959 - trunk/freenet/src/freenet/node In-Reply-To: <20080107200730.C0B203A13AC@freenetproject.org> References: <20080107200730.C0B203A13AC@freenetproject.org> Message-ID: <200801102310.10329.toad@amphibian.dyndns.org> With such an aggressive SEND_TIMEOUT, this borders on reviving NGRouting - it's a major change to routing, which since it hasn't been simulated will almost certainly cause major problems. No? On Monday 07 January 2008 20:07, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-07 20:07:30 +0000 (Mon, 07 Jan 2008) > New Revision: 16959 > > Modified: > trunk/freenet/src/freenet/node/MessageItem.java > trunk/freenet/src/freenet/node/PeerNode.java > trunk/freenet/src/freenet/node/RequestSender.java > Log: > implement conditionalSend: sendSync-with-timeout, aborts message send > > > Modified: trunk/freenet/src/freenet/node/MessageItem.java > =================================================================== > --- trunk/freenet/src/freenet/node/MessageItem.java 2008-01-07 19:31:24 UTC (rev 16958) > +++ trunk/freenet/src/freenet/node/MessageItem.java 2008-01-07 20:07:30 UTC (rev 16959) > @@ -72,4 +72,8 @@ > } > } > } > + > + public boolean isForMessage(Message msg) { > + return this.msg.equals(msg); > + } > } > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-07 19:31:24 UTC (rev 16958) > +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-07 20:07:30 UTC (rev 16959) > @@ -16,6 +16,7 @@ > import java.util.Hashtable; > import java.util.Iterator; > import java.util.LinkedList; > +import java.util.ListIterator; > import java.util.Vector; > import java.util.zip.DataFormatException; > import java.util.zip.Inflater; > @@ -968,6 +969,21 @@ > // It will wake up before the maximum coalescing delay (100ms) because > // it wakes up every 100ms *anyway*. > } > + > + private boolean maybeRemoveMessageFromQueue(Message removeMe) { > + Logger.normal(this, "attempting to remove message from send-queue: "+removeMe); > + synchronized (messagesToSendNow) { > + ListIterator i=messagesToSendNow.listIterator(); > + while (i.hasNext()) { > + MessageItem it=(MessageItem)i.next(); > + if (it.isForMessage(removeMe)) { > + i.remove(); > + return true; > + } > + } > + } > + return false; > + } > > public long getMessageQueueLengthBytes() { > long x = 0; > @@ -1396,6 +1412,36 @@ > } > > /** > + * Conceptually, send a message to this node IF it can be done within 'timeout', returing true > + * only after the message was sent (similiar to sendSync), and false if the message cannot be > + * sent to the node in that time period. As an optimization, however, this function may return > + * immediately if it is determined that the message would not leave the node within the timeout > + * period. > + */ > + public boolean conditionalSend(Message req, ByteCounter ctr, long timeout) throws NotConnectedException { > + if (timeout<=0) > + return false; > + if (getMessageQueueLengthBytes()/(getThrottle().getBandwidth()+1.0) > timeout) { > + Logger.normal(this, "conditionalSend; pre-emptively not sending message ("+timeout+"ms): "+req); > + return false; > + } > + SyncMessageCallback cb = new SyncMessageCallback(); > + sendAsync(req, cb, 0, ctr); > + cb.waitForSend(timeout); > + if (cb.done) { > + return true; > + } else { > + //best-effort: remove the message from the send queue it is ok if we can't prematurely > + //remove the item (i.e. race condition / now it is sent), but it will generate unclaimed messages, etc. > + if (!maybeRemoveMessageFromQueue(req)) > + Logger.error(this, "unable to stop transmition of request: "+req); > + else > + Logger.normal(this, "removed request from queue for timeout: "+req); > + return false; > + } > + } > + > + /** > * Enqueue a message to be sent to this node and wait up to a minute for it to be transmitted. > */ > public void sendSync(Message req, ByteCounter ctr) throws NotConnectedException { > @@ -1426,7 +1472,7 @@ > } > } > if(isConnected()) > - Logger.error(this, "Waited too long for a blocking send on " + this + " for " + PeerNode.this, new Exception("error")); > + Logger.normal(this, "Waited too long for a blocking send on " + this + " for " + PeerNode.this, new Exception("error")); > } > > public void acknowledged() { > > Modified: trunk/freenet/src/freenet/node/RequestSender.java > =================================================================== > --- trunk/freenet/src/freenet/node/RequestSender.java 2008-01-07 19:31:24 UTC (rev 16958) > +++ trunk/freenet/src/freenet/node/RequestSender.java 2008-01-07 20:07:30 UTC (rev 16959) > @@ -4,6 +4,7 @@ > package freenet.node; > > import java.util.HashSet; > +import java.util.ArrayList; > > import freenet.crypt.CryptFormatException; > import freenet.crypt.DSAPublicKey; > @@ -46,6 +47,8 @@ > public final class RequestSender implements Runnable, ByteCounter { > > // Constants > + //SEND_TIMEOUT is not a hard timeout, shoot low for low latency (250-500ms?). > + static final int SEND_TIMEOUT = 1000; > static final int ACCEPTED_TIMEOUT = 5000; > static final int FETCH_TIMEOUT = 120000; > /** Wait up to this long to get a path folding reply */ > @@ -141,6 +144,7 @@ > int rejectOverloads=0; > HashSet nodesRoutedTo = new HashSet(); > HashSet nodesNotIgnored = new HashSet(); > + ArrayList busyPeers = new ArrayList(); > while(true) { > if(logMINOR) Logger.minor(this, "htl="+htl); > if(htl == 0) { > @@ -159,9 +163,24 @@ > routeAttempts++; > > // Route it > + long sendTimeout = SEND_TIMEOUT; > + boolean usingBusyPeer=false; > PeerNode next; > next = node.peers.closerPeer(source, nodesRoutedTo, nodesNotIgnored, target, true, node.isAdvancedModeEnabled(), -1, null); > > + if (next == null && !busyPeers.isEmpty()) { > + next = (PeerNode)busyPeers.remove(0); > + usingBusyPeer=true; > + if (logMINOR) Logger.minor(this, "trying previously-found busy peer: "+next); > + //NOTE: if we are at this point, it is already presumed that the message cannot even make it off the node to this peer in SEND_TIMEOUT, use all the timeout we have left. > + sendTimeout = FETCH_TIMEOUT-(System.currentTimeMillis()-startTime); > + //Edge case, local request & we are running w/o any time left. > + if (sendTimeout < SEND_TIMEOUT && source==null) { > + if (logMINOR) Logger.minor(this, "increasing timeout for local request"); > + sendTimeout = 2*SEND_TIMEOUT; > + } > + } > + > if(next == null) { > if (logMINOR && rejectOverloads>0) > Logger.minor(this, "no more peers, but overloads ("+rejectOverloads+"/"+routeAttempts+" overloaded)"); > @@ -187,11 +206,18 @@ > // So take it from when we first started to try to send the request. > // See comments below when handling FNPRecentlyFailed for why we need this. > long timeSentRequest = System.currentTimeMillis(); > - > + > try { > //This is the first contact to this node > //async is preferred, but makes ACCEPTED_TIMEOUT much more likely for long send queues. > - next.sendAsync(req, null, 0, this); > + //using conditionalSend this way might actually approximate Q-routing load balancing accross the network. > + if (!next.conditionalSend(req, this, sendTimeout)) { > + if (usingBusyPeer) > + continue; > + Logger.normal(this, "will try this peer later if no others are available"); > + busyPeers.add(next); > + continue; > + } > } catch (NotConnectedException e) { > Logger.minor(this, "Not connected"); > continue; > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080110/ec063106/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 10 23:20:02 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 10 Jan 2008 23:20:02 +0000 Subject: [freenet-dev] [freenet-cvs] r16960 - trunk/freenet/src/freenet/node In-Reply-To: <20080107204520.4EAB139230A@freenetproject.org> References: <20080107204520.4EAB139230A@freenetproject.org> Message-ID: <200801102320.03182.toad@amphibian.dyndns.org> Just because our queue of data to send to a node is full doesn't mean its queue of data to send to us is full. Queue based load management would have to take the latter into account. In fact, it would have to take not only the data currently queued but also the data that is likely to become queued into account. The liability limiting code works on this principle. Various proposals for load management work similarly by passing back information on how many requests can be made in the near future - but these will not be deployed until they have been simulated. So please show me that this has no effect on routing, or revert it. I'd be happy to prioritise accepted's over data transfer in order to get an accepted more quickly; what priority e.g. requests should take is another question. When calculating such things we have to bear in mind that 95% of connections are asymmetrical, so the send queue and the receive queue may not be closely correlated. I'm also not entirely against what you've done here, but IMHO the code up to this commit is way too latency sensitive. If you just used ACCEPTED_TIMEOUT for example it would be a perfectly valid optimisation. I apologise for being away recently, some local problems. On Monday 07 January 2008 20:45, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-07 20:45:19 +0000 (Mon, 07 Jan 2008) > New Revision: 16960 > > Modified: > trunk/freenet/src/freenet/node/RequestSender.java > Log: > comment general theory of conditionalSend versus SEND_TIMEOUT > > > Modified: trunk/freenet/src/freenet/node/RequestSender.java > =================================================================== > --- trunk/freenet/src/freenet/node/RequestSender.java 2008-01-07 20:07:30 UTC (rev 16959) > +++ trunk/freenet/src/freenet/node/RequestSender.java 2008-01-07 20:45:19 UTC (rev 16960) > @@ -208,9 +208,27 @@ > long timeSentRequest = System.currentTimeMillis(); > > try { > - //This is the first contact to this node > - //async is preferred, but makes ACCEPTED_TIMEOUT much more likely for long send queues. > - //using conditionalSend this way might actually approximate Q-routing load balancing accross the network. > + //This is the first contact to this node, it is more likely to timeout > + /* > + * using sendSync could: > + * make ACCEPTED_TIMEOUT more accurate (as it is measured from the send-time), > + * use a lot of our time that we have to fulfill this request (simply waiting on the send queue, or longer if the node just went down), > + * using sendAsync could: > + * make ACCEPTED_TIMEOUT much more likely, > + * leave many hanging-requests/unclaimedFIFO items, > + * potentially make overloaded peers MORE overloaded (we make a request and promptly forget about them). > + * using conditionalSend could: > + * make ACCEPTED_TIMEOUT as accurate as sendSync (as it to waits for transmittion) > + * reduce general latency around peers which have slow network links > + * not needlessly overload nodes w/ forgotten requests (as conditonalSend will try and withdraw the request if it times out) > + *!!!make us skip peers which would otherwise have the data (they are closer, but slower) > + * > + * To avoid the pitfall of conditionalSend (potentially skipping a good peer), we will come back to them when it is > + * apparent that we cannot fill the request quickly. Using conditionalSend this way might actually approximate > + * Q-routing (for load balancing/latency) across the network; if SEND_TIMEOUT is too high... this reduces to > + * using sendSync w/ a good error catch, and if SEND_TIMEOUT is too low... this reduces to creating a cache-backbone > + * of fast links in the network which will always be queried before general nodes in the network are. > + */ > if (!next.conditionalSend(req, this, sendTimeout)) { > if (usingBusyPeer) > continue; > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080110/b3d3b7e3/attachment.pgp From toad at amphibian.dyndns.org Thu Jan 10 23:23:31 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 10 Jan 2008 23:23:31 +0000 Subject: [freenet-dev] [freenet-cvs] r16962 - trunk/freenet/src/freenet/io/xfer In-Reply-To: <20080108004635.8B11547AED4@freenetproject.org> References: <20080108004635.8B11547AED4@freenetproject.org> Message-ID: <200801102323.31662.toad@amphibian.dyndns.org> On Tuesday 08 January 2008 00:46, you wrote: > Author: robert > Date: 2008-01-08 00:46:35 +0000 (Tue, 08 Jan 2008) > New Revision: 16962 > > Modified: > trunk/freenet/src/freenet/io/xfer/BlockReceiver.java > trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > Log: > trivial: comments, logging, readability Doesn't look that way to me e.g. timeAllSent affects when we kill the transfer. What's going on here? > > > Modified: trunk/freenet/src/freenet/io/xfer/BlockReceiver.java > =================================================================== > --- trunk/freenet/src/freenet/io/xfer/BlockReceiver.java 2008-01-07 20:56:43 UTC (rev 16961) > +++ trunk/freenet/src/freenet/io/xfer/BlockReceiver.java 2008-01-08 00:46:35 UTC (rev 16962) > @@ -66,6 +66,7 @@ > > public byte[] receive() throws RetrievalException { > int consecutiveMissingPacketReports = 0; > + boolean logMINOR=Logger.shouldLog(Logger.MINOR, this); > try { > while (!_prb.allReceived()) { > Message m1; > @@ -80,7 +81,7 @@ > _prb.abort(RetrievalException.SENDER_DIED, "Disconnected during receive"); > throw new RetrievalException(RetrievalException.SENDER_DISCONNECTED); > } > - if(Logger.shouldLog(Logger.MINOR, this)) > + if(logMINOR) > Logger.minor(this, "Received "+m1); > if ((m1 != null) && m1.getSpec().equals(DMT.sendAborted)) { > _prb.abort(m1.getInt(DMT.REASON), m1.getString(DMT.DESCRIPTION)); > @@ -112,7 +113,7 @@ > } > } > } > - if(Logger.shouldLog(Logger.MINOR, this)) > + if(logMINOR) > Logger.minor(this, "Missing: "+missing.size()); > if (missing.size() > 0) { > Message mn = DMT.createMissingPacketNotification(_uid, missing); > > Modified: trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > =================================================================== > --- trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java 2008-01-07 20:56:43 UTC (rev 16961) > +++ trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java 2008-01-08 00:46:35 UTC (rev 16962) > @@ -84,9 +84,13 @@ > try { > while (true) { > synchronized(_senderThread) { > - if(_unsent.size() != 0) break; > + if(_unsent.size() != 0) { > + timeAllSent = -1; > + break; > + } > // No unsent packets > if(getNumSent() == _prb.getNumPackets()) { > + //No unreceived packets > if(Logger.shouldLog(Logger.MINOR, this)) > Logger.minor(this, "Sent all blocks, none unsent"); > if(timeAllSent <= 0) > @@ -96,7 +100,6 @@ > _senderThread.wait(10*1000); > } > } > - timeAllSent = -1; > } catch (InterruptedException e) { > } catch (AbortedException e) { > synchronized(_senderThread) { > @@ -136,9 +139,8 @@ > } > } > > - /** @return True if _sendComplete */ > private void delay(long startCycleTime) { > - > + //FIXME: startCycleTime is not used in this function, why is it passed in? > long startThrottle = System.currentTimeMillis(); > > // Get the current inter-packet delay > @@ -164,6 +166,7 @@ > int x = (int) (Math.min(l, 120*1000)); > if(x > 0) { > try { > + //FIXME: if the senderThread sleeps here for two minutes, that will timeout the receiver, no? Should this be a wait()? > Thread.sleep(x); > } catch (InterruptedException e) { > // Ignore > @@ -239,6 +242,7 @@ > return false; > if (msg == null) { > long now = System.currentTimeMillis(); > + //SEND_TIMEOUT (one minute) after all packets have been transmitted, terminate the send. > if((timeAllSent > 0) && ((now - timeAllSent) > SEND_TIMEOUT) && > (getNumSent() == _prb.getNumPackets())) { > synchronized(_senderThread) { > @@ -280,9 +284,8 @@ > _senderThread.notifyAll(); > } > return false; > - } else if(_sendComplete) { > - // Terminated abnormally > - return false; > + } else { > + Logger.error(this, "Transmitter received unknown message type: "+msg.getSpec().getName()); > } > } > } catch (AbortedException e) { > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080110/a9dd1639/attachment.pgp From robert at emu.freenetproject.org Thu Jan 10 23:31:59 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Thu, 10 Jan 2008 17:31:59 -0600 Subject: [freenet-dev] [freenet-cvs] r16962 - trunk/freenet/src/freenet/io/xfer In-Reply-To: <200801102323.31662.toad@amphibian.dyndns.org> References: <20080108004635.8B11547AED4@freenetproject.org> <200801102323.31662.toad@amphibian.dyndns.org> Message-ID: <9BE53537-882D-4456-B85F-41564FE192D7@emu.freenetproject.org> On Jan 10, 2008, at 5:23 PM, Matthew Toseland wrote: > On Tuesday 08 January 2008 00:46, you wrote: >> Author: robert >> Date: 2008-01-08 00:46:35 +0000 (Tue, 08 Jan 2008) >> New Revision: 16962 >> >> Modified: >> trunk/freenet/src/freenet/io/xfer/BlockReceiver.java >> trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java >> Log: >> trivial: comments, logging, readability > > Doesn't look that way to me e.g. timeAllSent affects when we kill the > transfer. What's going on here? There was more than one problem with timeAllSent... there are a number of commits regarding the block transmittion/reception. This change actually is trivial because the only time that timeAllSent was set is on breaking out of the loop, so I moved it to the break statement so that one can see that. -- Robert Hailey > >> >> >> Modified: trunk/freenet/src/freenet/io/xfer/BlockReceiver.java >> =================================================================== >> --- trunk/freenet/src/freenet/io/xfer/BlockReceiver.java 2008-01-07 >> 20:56:43 > UTC (rev 16961) >> +++ trunk/freenet/src/freenet/io/xfer/BlockReceiver.java 2008-01-08 >> 00:46:35 > UTC (rev 16962) >> @@ -66,6 +66,7 @@ >> >> public byte[] receive() throws RetrievalException { >> int consecutiveMissingPacketReports = 0; >> + boolean logMINOR=Logger.shouldLog(Logger.MINOR, this); >> try { >> while (!_prb.allReceived()) { >> Message m1; >> @@ -80,7 +81,7 @@ >> _prb.abort(RetrievalException.SENDER_DIED, >> "Disconnected > during receive"); >> throw new > RetrievalException(RetrievalException.SENDER_DISCONNECTED); >> } >> - if(Logger.shouldLog(Logger.MINOR, this)) >> + if(logMINOR) >> Logger.minor(this, "Received "+m1); >> if ((m1 != null) && >> m1.getSpec().equals(DMT.sendAborted)) { >> _prb.abort(m1.getInt(DMT.REASON), m1.getString(DMT.DESCRIPTION)); >> @@ -112,7 +113,7 @@ >> } >> } >> } >> - if(Logger.shouldLog(Logger.MINOR, this)) >> + if(logMINOR) >> Logger.minor(this, "Missing: "+missing.size()); >> if (missing.size() > 0) { >> Message mn = DMT.createMissingPacketNotification(_uid, missing); >> >> Modified: trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java >> =================================================================== >> --- trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java >> 2008-01-07 > 20:56:43 UTC (rev 16961) >> +++ trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java >> 2008-01-08 > 00:46:35 UTC (rev 16962) >> @@ -84,9 +84,13 @@ >> try { >> while (true) { >> synchronized(_senderThread) { >> - if(_unsent.size() != 0) break; >> + if(_unsent.size() != 0) { >> + timeAllSent = -1; >> + break; >> + } >> // No unsent packets >> if(getNumSent() == _prb.getNumPackets()) { >> + //No unreceived packets >> if(Logger.shouldLog(Logger.MINOR, this)) >> Logger.minor(this, "Sent all blocks, none unsent"); >> if(timeAllSent <= 0) >> @@ -96,7 +100,6 @@ >> _senderThread.wait(10*1000); >> } >> } >> - timeAllSent = -1; >> } catch (InterruptedException e) { >> } catch (AbortedException e) { >> synchronized(_senderThread) { >> @@ -136,9 +139,8 @@ >> } >> } >> >> - /** @return True if _sendComplete */ >> private void delay(long startCycleTime) { >> - >> + //FIXME: startCycleTime is not used in this function, why is >> it passed > in? >> long startThrottle = System.currentTimeMillis(); >> >> // Get the current inter-packet delay >> @@ -164,6 +166,7 @@ >> int x = (int) (Math.min(l, 120*1000)); >> if(x > 0) { >> try { >> + //FIXME: if the senderThread sleeps here for two minutes, >> that will > timeout the receiver, no? Should this be a wait()? >> Thread.sleep(x); >> } catch (InterruptedException e) { >> // Ignore >> @@ -239,6 +242,7 @@ >> return false; >> if (msg == null) { >> long now = System.currentTimeMillis(); >> + //SEND_TIMEOUT (one minute) after all packets have been >> transmitted, > terminate the send. >> if((timeAllSent > 0) && ((now - timeAllSent) > SEND_TIMEOUT) && >> (getNumSent() == _prb.getNumPackets())) { >> synchronized(_senderThread) { >> @@ -280,9 +284,8 @@ >> _senderThread.notifyAll(); >> } >> return false; >> - } else if(_sendComplete) { >> - // Terminated abnormally >> - return false; >> + } else { >> + Logger.error(this, "Transmitter received unknown message > type: "+msg.getSpec().getName()); >> } >> } >> } catch (AbortedException e) { >> >> _______________________________________________ >> cvs mailing list >> cvs at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs >> >> > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From robert at emu.freenetproject.org Thu Jan 10 23:35:31 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Thu, 10 Jan 2008 17:35:31 -0600 Subject: [freenet-dev] [freenet-cvs] r16959 - trunk/freenet/src/freenet/node In-Reply-To: <200801102310.10329.toad@amphibian.dyndns.org> References: <20080107200730.C0B203A13AC@freenetproject.org> <200801102310.10329.toad@amphibian.dyndns.org> Message-ID: On Jan 10, 2008, at 5:10 PM, Matthew Toseland wrote: > With such an aggressive SEND_TIMEOUT, this borders on reviving > NGRouting - > it's a major change to routing, which since it hasn't been simulated > will > almost certainly cause major problems. No? I'm thinking that the SEND_TIMEOUT will need to be increased, I really am not familiar with what the average network condition will be like. Feel free to change the SEND_TIMEOUTs to your liking. Also, I noticed that the htl is still decremented if we skip a peer w/ conditionalSend (or on disconnect). On Jan 10, 2008, at 5:20 PM, Matthew Toseland wrote: > Just because our queue of data to send to a node is full doesn't > mean its > queue of data to send to us is full. Queue based load management > would have > to take the latter into account. In fact, it would have to take not > only the > data currently queued but also the data that is likely to become > queued into > account. The liability limiting code works on this principle. Various > proposals for load management work similarly by passing back > information on > how many requests can be made in the near future - but these will > not be > deployed until they have been simulated. > > So please show me that this has no effect on routing, or revert it. My initial impression was that with SEND_TIMEOUT being either ACCEPTED_TIMEOUT (equivalent to sendAsync) or one minute (equivalent to sendSync) that it would be the same. It does change the behavior though (as the packet is withdrawn from the send queue, or not sent). Until it can be simulated, I can simply flip the send back to sendSync(). -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080110/8df4c446/attachment.htm From toad at amphibian.dyndns.org Thu Jan 10 23:54:06 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 10 Jan 2008 23:54:06 +0000 Subject: [freenet-dev] [freenet-cvs] r16974 - trunk/freenet/src/freenet/node In-Reply-To: <20080109005307.94EB747A0E3@freenetproject.org> References: <20080109005307.94EB747A0E3@freenetproject.org> Message-ID: <200801102354.10975.toad@amphibian.dyndns.org> On Wednesday 09 January 2008 00:53, zothar at freenetproject.org wrote: > Author: zothar > Date: 2008-01-09 00:53:07 +0000 (Wed, 09 Jan 2008) > New Revision: 16974 > > Modified: > trunk/freenet/src/freenet/node/DarknetPeerNode.java > trunk/freenet/src/freenet/node/OpennetPeerNode.java > trunk/freenet/src/freenet/node/PeerNode.java > trunk/freenet/src/freenet/node/SeedClientPeerNode.java > trunk/freenet/src/freenet/node/SeedServerPeerNode.java > Log: > Implement isDarknet(). Move sendNodeToNodeMessage() to PeerNode and queue on not connected only if the peer is a darknet peer. This is getting awfully complicated. I had assumed that N2NMs were a mechanism for clients (including fproxy, which is an internal client) to communicate with each other. N2NMs are therefore only useful on darknet, since on opennet there is no reason to talk to other local clients - certainly this reasoning applies to fproxy text messages, and certainly it applies to queueing messages. On the other hand, there may be some applications for which being able to talk to your opposite client on your opennet peers may be useful - namely any application that implements its own routed protocol on top of Freenet. Many of these (e.g. currency over freenet) require a trust connection - but maybe some wouldn't (e.g. broadcasting). I had assumed that not providing any N2NM service for opennet peers was a sensible default, which we could change when a client author asks for it. But perhaps I was wrong? One obvious limitation with allowing N2NMs to/from opennet peers is that queueing is probably a bad idea. However, once we have FCP clients, we might want to provide a way to send an N2NM and not queue it if the node is disconnected, just drop it, because it's not important if it's not delivered quickly. This can easily be implemented on top of queueing. So the simplest solution is to always queue, to only allow N2NMs to/from darknet peers, and to have a separate message type for the low-level mechanism of differential noderefs (which can of course share code if there is any major code to be done). Is the simplest solution the best in this case? It seems like premature complexity to me - but maybe it's useful complexity? > > Modified: trunk/freenet/src/freenet/node/DarknetPeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/DarknetPeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/DarknetPeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -1257,28 +1257,6 @@ > } > } > > - public void sendNodeToNodeMessage(SimpleFieldSet fs, int n2nType, boolean includeSentTime, long now) { > - fs.put("n2nType", n2nType); > - if(includeSentTime) { > - fs.put("sentTime", now); > - } > - try { > - Message n2nm; > - n2nm = DMT.createNodeToNodeMessage( > - n2nType, fs.toString().getBytes("UTF-8")); > - try { > - sendAsync(n2nm, null, 0, null); > - } catch (NotConnectedException e) { > - if(includeSentTime) { > - fs.removeValue("sentTime"); > - } > - queueN2NM(fs); > - } > - } catch (UnsupportedEncodingException e) { > - throw new Error("Impossible: "+e, e); > - } > - } > - > public int sendFileOfferAccepted(long uid) { > storeOffers(); > long now = System.currentTimeMillis(); > @@ -1509,6 +1487,10 @@ > return new DarknetPeerNodeStatus(this, noHeavy); > } > > + public boolean isDarknet() { > + return true; > + } > + > public boolean isOpennet() { > return false; > } > > Modified: trunk/freenet/src/freenet/node/OpennetPeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/OpennetPeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/OpennetPeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -23,6 +23,10 @@ > return super.isRoutingCompatible(); > } > > + public boolean isDarknet() { > + return false; > + } > + > public boolean isOpennet() { > return true; > } > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -2367,6 +2367,8 @@ > return fs; > } > > + public abstract boolean isDarknet(); > + > public abstract boolean isOpennet(); > > /** > @@ -3498,4 +3500,35 @@ > } > > private int handshakeIPAlternator; > + > + public void sendNodeToNodeMessage(SimpleFieldSet fs, int n2nType, boolean includeSentTime, long now) { > + fs.put("n2nType", n2nType); > + if(includeSentTime) { > + fs.put("sentTime", now); > + } > + try { > + Message n2nm; > + n2nm = DMT.createNodeToNodeMessage( > + n2nType, fs.toString().getBytes("UTF-8")); > + try { > + sendAsync(n2nm, null, 0, null); > + } catch (NotConnectedException e) { > + if(includeSentTime) { > + fs.removeValue("sentTime"); > + } > + if(isDarknet()) { > + queueN2NM(fs); > + } > + } > + } catch (UnsupportedEncodingException e) { > + throw new Error("Impossible: "+e, e); > + } > + } > + > + /** > + * A method to be to queue an N2NM in a extra peer data file, only implemented by DarknetPeerNode > + */ > + public void queueN2NM(SimpleFieldSet fs) { > + // Do nothing in the default impl > + } > } > > Modified: trunk/freenet/src/freenet/node/SeedClientPeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/SeedClientPeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/SeedClientPeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -21,6 +21,10 @@ > return new PeerNodeStatus(this, noHeavy); > } > > + public boolean isDarknet() { > + return false; > + } > + > public boolean isOpennet() { > return false; // Not exactly > } > > Modified: trunk/freenet/src/freenet/node/SeedServerPeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/SeedServerPeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/SeedServerPeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -26,6 +26,10 @@ > return new PeerNodeStatus(this, noHeavy); > } > > + public boolean isDarknet() { > + return false; > + } > + > public boolean isOpennet() { > return false; > } > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080110/2da9a894/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 00:15:02 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 00:15:02 +0000 Subject: [freenet-dev] [freenet-cvs] r16974 - trunk/freenet/src/freenet/node In-Reply-To: <20080109005307.94EB747A0E3@freenetproject.org> References: <20080109005307.94EB747A0E3@freenetproject.org> Message-ID: <200801110015.16489.toad@amphibian.dyndns.org> Is there any reason to have both isDarknet() and isOpennet()? I suppose seed nodes would return false for both? On Wednesday 09 January 2008 00:53, zothar at freenetproject.org wrote: > Author: zothar > Date: 2008-01-09 00:53:07 +0000 (Wed, 09 Jan 2008) > New Revision: 16974 > > Modified: > trunk/freenet/src/freenet/node/DarknetPeerNode.java > trunk/freenet/src/freenet/node/OpennetPeerNode.java > trunk/freenet/src/freenet/node/PeerNode.java > trunk/freenet/src/freenet/node/SeedClientPeerNode.java > trunk/freenet/src/freenet/node/SeedServerPeerNode.java > Log: > Implement isDarknet(). Move sendNodeToNodeMessage() to PeerNode and queue on not connected only if the peer is a darknet peer. > > Modified: trunk/freenet/src/freenet/node/DarknetPeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/DarknetPeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/DarknetPeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -1257,28 +1257,6 @@ > } > } > > - public void sendNodeToNodeMessage(SimpleFieldSet fs, int n2nType, boolean includeSentTime, long now) { > - fs.put("n2nType", n2nType); > - if(includeSentTime) { > - fs.put("sentTime", now); > - } > - try { > - Message n2nm; > - n2nm = DMT.createNodeToNodeMessage( > - n2nType, fs.toString().getBytes("UTF-8")); > - try { > - sendAsync(n2nm, null, 0, null); > - } catch (NotConnectedException e) { > - if(includeSentTime) { > - fs.removeValue("sentTime"); > - } > - queueN2NM(fs); > - } > - } catch (UnsupportedEncodingException e) { > - throw new Error("Impossible: "+e, e); > - } > - } > - > public int sendFileOfferAccepted(long uid) { > storeOffers(); > long now = System.currentTimeMillis(); > @@ -1509,6 +1487,10 @@ > return new DarknetPeerNodeStatus(this, noHeavy); > } > > + public boolean isDarknet() { > + return true; > + } > + > public boolean isOpennet() { > return false; > } > > Modified: trunk/freenet/src/freenet/node/OpennetPeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/OpennetPeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/OpennetPeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -23,6 +23,10 @@ > return super.isRoutingCompatible(); > } > > + public boolean isDarknet() { > + return false; > + } > + > public boolean isOpennet() { > return true; > } > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -2367,6 +2367,8 @@ > return fs; > } > > + public abstract boolean isDarknet(); > + > public abstract boolean isOpennet(); > > /** > @@ -3498,4 +3500,35 @@ > } > > private int handshakeIPAlternator; > + > + public void sendNodeToNodeMessage(SimpleFieldSet fs, int n2nType, boolean includeSentTime, long now) { > + fs.put("n2nType", n2nType); > + if(includeSentTime) { > + fs.put("sentTime", now); > + } > + try { > + Message n2nm; > + n2nm = DMT.createNodeToNodeMessage( > + n2nType, fs.toString().getBytes("UTF-8")); > + try { > + sendAsync(n2nm, null, 0, null); > + } catch (NotConnectedException e) { > + if(includeSentTime) { > + fs.removeValue("sentTime"); > + } > + if(isDarknet()) { > + queueN2NM(fs); > + } > + } > + } catch (UnsupportedEncodingException e) { > + throw new Error("Impossible: "+e, e); > + } > + } > + > + /** > + * A method to be to queue an N2NM in a extra peer data file, only implemented by DarknetPeerNode > + */ > + public void queueN2NM(SimpleFieldSet fs) { > + // Do nothing in the default impl > + } > } > > Modified: trunk/freenet/src/freenet/node/SeedClientPeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/SeedClientPeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/SeedClientPeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -21,6 +21,10 @@ > return new PeerNodeStatus(this, noHeavy); > } > > + public boolean isDarknet() { > + return false; > + } > + > public boolean isOpennet() { > return false; // Not exactly > } > > Modified: trunk/freenet/src/freenet/node/SeedServerPeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/SeedServerPeerNode.java 2008-01-09 00:00:39 UTC (rev 16973) > +++ trunk/freenet/src/freenet/node/SeedServerPeerNode.java 2008-01-09 00:53:07 UTC (rev 16974) > @@ -26,6 +26,10 @@ > return new PeerNodeStatus(this, noHeavy); > } > > + public boolean isDarknet() { > + return false; > + } > + > public boolean isOpennet() { > return false; > } > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/5bb30078/attachment.pgp From m.rogers at cs.ucl.ac.uk Fri Jan 11 00:28:47 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 11 Jan 2008 00:28:47 +0000 Subject: [freenet-dev] [freenet-cvs] r16974 - trunk/freenet/src/freenet/node In-Reply-To: <200801102354.10975.toad@amphibian.dyndns.org> References: <20080109005307.94EB747A0E3@freenetproject.org> <200801102354.10975.toad@amphibian.dyndns.org> Message-ID: <4786B83F.6010406@cs.ucl.ac.uk> Matthew Toseland wrote: > I had assumed that N2NMs were a mechanism for clients (including fproxy, which > is an internal client) to communicate with each other. N2NMs are therefore > only useful on darknet, since on opennet there is no reason to talk to other > local clients - I was going to suggest N2N chat between opennet peers, perhaps with an eye to establishing a darknet connection. But there's a danger that, since it's Freenet, users will assume they're anonymous when chatting... and there doesn't seem much point offering a friendly service like N2N chat if it has to be plastered in warnings. Cheers, Michael From robert at emu.freenetproject.org Fri Jan 11 02:20:34 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Thu, 10 Jan 2008 20:20:34 -0600 Subject: [freenet-dev] [freenet-cvs] r16960 - trunk/freenet/src/freenet/node In-Reply-To: <200801102320.03182.toad@amphibian.dyndns.org> References: <20080107204520.4EAB139230A@freenetproject.org> <200801102320.03182.toad@amphibian.dyndns.org> Message-ID: <0E9BEE4B-DA52-4EDC-9508-B7F2003B62B8@emu.freenetproject.org> On Jan 10, 2008, at 5:20 PM, Matthew Toseland wrote: > I'm also not entirely against what you've done here, but IMHO the > code up to > this commit is way too latency sensitive. If you just used > ACCEPTED_TIMEOUT > for example it would be a perfectly valid optimisation. I think that was my major misunderstanding. While it is for certain that the faster requests are processed the "better" the network at large, I was running under the assumption that if the most-recently- routed-to-node did not process the request in FETCH_TIMEOUT that the world was over, and nothing would ever work. Concerning the first request made to a node, both sendSync and sendAsync seem interestingly wrong for this use. * Surely if we use sendAsync, once the average send queue reaches the ACCEPTED_TIMEOUT/2, the network is disastrously flooded as the nodes all send every request to all there nodes waiting on none of them. * Intuitively, sendSync *seems* to be better. But again, once the average send queue reaches ACCEPTED_TIMEOUT, the accepted packet BACK to the originating node will be lost (and it will continue), and the requesting node will go about it's business spreading the request. That is how I came up with the conditionalSend, but at best it only attacks half the problem (exactly as you said). > I'd be happy to prioritise accepted's over data transfer in order to > get an > accepted more quickly; what priority e.g. requests should take is > another > question. When calculating such things we have to bear in mind that > 95% of > connections are asymmetrical, so the send queue and the receive > queue may not > be closely correlated. Prioritizing accepted messages over bulk data may help this case, but the more general solution would be to prioritize data less than control packets, no? Find the data fast (Request/DF/DNF/RNF/...), transfer it slow (packetTransmit/???). Then this issue would only come back up if the time to transmit the control messages in the queue was more than ACCEPTED_TIMEOUT/etc. I do not reason there would be any issue with the requests being the same (higher) priority, as even with a huge back-logged send queue of data the standard reject mechanism would be effective (send queue length/threadLimit), we would just know about it much sooner. In fact... I *REALLY* like that solution. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080110/5d8843b8/attachment.htm From m.rogers at cs.ucl.ac.uk Fri Jan 11 03:47:20 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 11 Jan 2008 03:47:20 +0000 Subject: [freenet-dev] [freenet-cvs] r16960 - trunk/freenet/src/freenet/node In-Reply-To: <0E9BEE4B-DA52-4EDC-9508-B7F2003B62B8@emu.freenetproject.org> References: <20080107204520.4EAB139230A@freenetproject.org> <200801102320.03182.toad@amphibian.dyndns.org> <0E9BEE4B-DA52-4EDC-9508-B7F2003B62B8@emu.freenetproject.org> Message-ID: <4786E6C8.90506@cs.ucl.ac.uk> Robert Hailey wrote: > Prioritizing accepted messages over bulk data may help this case, but > the more general solution would be to prioritize data less than control > packets, no? Find the data fast (Request/DF/DNF/RNF/...), transfer it > slow (packetTransmit/???). We'd have to be careful not to let the low-priority traffic starve, otherwise we'd keep accepting new requests while existing transfers repeatedly got pushed to the back of the queue and eventually timed out. Cheers, Michael From robert at emu.freenetproject.org Fri Jan 11 06:06:43 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 00:06:43 -0600 Subject: [freenet-dev] [freenet-cvs] r16960 - trunk/freenet/src/freenet/node In-Reply-To: <4786E6C8.90506@cs.ucl.ac.uk> References: <20080107204520.4EAB139230A@freenetproject.org> <200801102320.03182.toad@amphibian.dyndns.org> <0E9BEE4B-DA52-4EDC-9508-B7F2003B62B8@emu.freenetproject.org> <4786E6C8.90506@cs.ucl.ac.uk> Message-ID: <20080111000643.3ptzwun1mskc84gc@69.49.230.6> Quoting Michael Rogers : > Robert Hailey wrote: >> Prioritizing accepted messages over bulk data may help this case, but >> the more general solution would be to prioritize data less than control >> packets, no? Find the data fast (Request/DF/DNF/RNF/...), transfer it >> slow (packetTransmit/???). > > We'd have to be careful not to let the low-priority traffic starve, > otherwise we'd keep accepting new requests while existing transfers > repeatedly got pushed to the back of the queue and eventually timed out. > > Cheers, > Michael I disagree. I think that it should be a strict priority (if there is any queued control packets they are sent first). Both the number of pending requests (represented by thread count if nothing else) and the data send-queue length are parameters for rejection. Rejecting new requests would keep the data-transfer level traffic from being starved. If my understanding of how small (and therefore easily coalescable) these control packets are is correct, there would be no chance of starvation. And it only makes sense... the data packets want high throughput and the control packets want low latency. I imagine once this is implemented data searches timing out will be a thing of the past. -- Robert Hailey From m.rogers at cs.ucl.ac.uk Fri Jan 11 08:34:17 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 11 Jan 2008 08:34:17 +0000 Subject: [freenet-dev] [freenet-cvs] r16960 - trunk/freenet/src/freenet/node In-Reply-To: <20080111000643.3ptzwun1mskc84gc@69.49.230.6> References: <20080107204520.4EAB139230A@freenetproject.org> <200801102320.03182.toad@amphibian.dyndns.org> <0E9BEE4B-DA52-4EDC-9508-B7F2003B62B8@emu.freenetproject.org> <4786E6C8.90506@cs.ucl.ac.uk> <20080111000643.3ptzwun1mskc84gc@69.49.230.6> Message-ID: <47872A09.40109@cs.ucl.ac.uk> Robert Hailey wrote: > Rejecting new > requests would keep the data-transfer level traffic from being > starved. Yup, that makes sense as long as it also applies to local requests. Cheers, Michael From toad at amphibian.dyndns.org Fri Jan 11 16:50:10 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 16:50:10 +0000 Subject: [freenet-dev] [freenet-cvs] r16962 - trunk/freenet/src/freenet/io/xfer In-Reply-To: <9BE53537-882D-4456-B85F-41564FE192D7@emu.freenetproject.org> References: <20080108004635.8B11547AED4@freenetproject.org> <200801102323.31662.toad@amphibian.dyndns.org> <9BE53537-882D-4456-B85F-41564FE192D7@emu.freenetproject.org> Message-ID: <200801111650.15354.toad@amphibian.dyndns.org> On Thursday 10 January 2008 23:31, Robert Hailey wrote: > > On Jan 10, 2008, at 5:23 PM, Matthew Toseland wrote: > > > On Tuesday 08 January 2008 00:46, you wrote: > >> Author: robert > >> Date: 2008-01-08 00:46:35 +0000 (Tue, 08 Jan 2008) > >> New Revision: 16962 > >> > >> Modified: > >> trunk/freenet/src/freenet/io/xfer/BlockReceiver.java > >> trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > >> Log: > >> trivial: comments, logging, readability > > > > Doesn't look that way to me e.g. timeAllSent affects when we kill the > > transfer. What's going on here? > > There was more than one problem with timeAllSent... there are a number > of commits regarding the block transmittion/reception. This change > actually is trivial because the only time that timeAllSent was set is > on breaking out of the loop, so I moved it to the break statement so > that one can see that. And what about removing else if(_allSent) { ... return } ? > > -- > Robert Hailey > > > > >> > >> > >> Modified: trunk/freenet/src/freenet/io/xfer/BlockReceiver.java > >> =================================================================== > >> --- trunk/freenet/src/freenet/io/xfer/BlockReceiver.java 2008-01-07 > >> 20:56:43 > > UTC (rev 16961) > >> +++ trunk/freenet/src/freenet/io/xfer/BlockReceiver.java 2008-01-08 > >> 00:46:35 > > UTC (rev 16962) > >> @@ -66,6 +66,7 @@ > >> > >> public byte[] receive() throws RetrievalException { > >> int consecutiveMissingPacketReports = 0; > >> + boolean logMINOR=Logger.shouldLog(Logger.MINOR, this); > >> try { > >> while (!_prb.allReceived()) { > >> Message m1; > >> @@ -80,7 +81,7 @@ > >> _prb.abort(RetrievalException.SENDER_DIED, > >> "Disconnected > > during receive"); > >> throw new > > RetrievalException(RetrievalException.SENDER_DISCONNECTED); > >> } > >> - if(Logger.shouldLog(Logger.MINOR, this)) > >> + if(logMINOR) > >> Logger.minor(this, "Received "+m1); > >> if ((m1 != null) && > >> m1.getSpec().equals(DMT.sendAborted)) { > >> _prb.abort(m1.getInt(DMT.REASON), m1.getString(DMT.DESCRIPTION)); > >> @@ -112,7 +113,7 @@ > >> } > >> } > >> } > >> - if(Logger.shouldLog(Logger.MINOR, this)) > >> + if(logMINOR) > >> Logger.minor(this, "Missing: "+missing.size()); > >> if (missing.size() > 0) { > >> Message mn = DMT.createMissingPacketNotification(_uid, missing); > >> > >> Modified: trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > >> =================================================================== > >> --- trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > >> 2008-01-07 > > 20:56:43 UTC (rev 16961) > >> +++ trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > >> 2008-01-08 > > 00:46:35 UTC (rev 16962) > >> @@ -84,9 +84,13 @@ > >> try { > >> while (true) { > >> synchronized(_senderThread) { > >> - if(_unsent.size() != 0) break; > >> + if(_unsent.size() != 0) { > >> + timeAllSent = -1; > >> + break; > >> + } > >> // No unsent packets > >> if(getNumSent() == _prb.getNumPackets()) { > >> + //No unreceived packets > >> if(Logger.shouldLog(Logger.MINOR, this)) > >> Logger.minor(this, "Sent all blocks, none unsent"); > >> if(timeAllSent <= 0) > >> @@ -96,7 +100,6 @@ > >> _senderThread.wait(10*1000); > >> } > >> } > >> - timeAllSent = -1; > >> } catch (InterruptedException e) { > >> } catch (AbortedException e) { > >> synchronized(_senderThread) { > >> @@ -136,9 +139,8 @@ > >> } > >> } > >> > >> - /** @return True if _sendComplete */ > >> private void delay(long startCycleTime) { > >> - > >> + //FIXME: startCycleTime is not used in this function, why is > >> it passed > > in? > >> long startThrottle = System.currentTimeMillis(); > >> > >> // Get the current inter-packet delay > >> @@ -164,6 +166,7 @@ > >> int x = (int) (Math.min(l, 120*1000)); > >> if(x > 0) { > >> try { > >> + //FIXME: if the senderThread sleeps here for two minutes, > >> that will > > timeout the receiver, no? Should this be a wait()? > >> Thread.sleep(x); > >> } catch (InterruptedException e) { > >> // Ignore > >> @@ -239,6 +242,7 @@ > >> return false; > >> if (msg == null) { > >> long now = System.currentTimeMillis(); > >> + //SEND_TIMEOUT (one minute) after all packets have been > >> transmitted, > > terminate the send. > >> if((timeAllSent > 0) && ((now - timeAllSent) > SEND_TIMEOUT) && > >> (getNumSent() == _prb.getNumPackets())) { > >> synchronized(_senderThread) { > >> @@ -280,9 +284,8 @@ > >> _senderThread.notifyAll(); > >> } > >> return false; > >> - } else if(_sendComplete) { > >> - // Terminated abnormally > >> - return false; > >> + } else { > >> + Logger.error(this, "Transmitter received unknown message > > type: "+msg.getSpec().getName()); > >> } > >> } > >> } catch (AbortedException e) { > >> > >> _______________________________________________ > >> cvs mailing list > >> cvs at freenetproject.org > >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > >> > >> > > _______________________________________________ > > Devl mailing list > > Devl at freenetproject.org > > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/927b25c2/attachment.pgp From robert at emu.freenetproject.org Fri Jan 11 17:17:21 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 11:17:21 -0600 Subject: [freenet-dev] [freenet-cvs] r16962 - trunk/freenet/src/freenet/io/xfer In-Reply-To: <200801111650.15354.toad@amphibian.dyndns.org> References: <20080108004635.8B11547AED4@freenetproject.org> <200801102323.31662.toad@amphibian.dyndns.org> <9BE53537-882D-4456-B85F-41564FE192D7@emu.freenetproject.org> <200801111650.15354.toad@amphibian.dyndns.org> Message-ID: <17262D8E-8EBE-42A5-A09A-EE134C3A065E@emu.freenetproject.org> On Jan 11, 2008, at 10:50 AM, Matthew Toseland wrote: >>>> Log: >>>> trivial: comments, logging, readability >>> >>> Doesn't look that way to me e.g. timeAllSent affects when we kill >>> the >>> transfer. What's going on here? >> >> There was more than one problem with timeAllSent... there are a >> number >> of commits regarding the block transmittion/reception. This change >> actually is trivial because the only time that timeAllSent was set is >> on breaking out of the loop, so I moved it to the break statement so >> that one can see that. > > And what about removing else if(_allSent) { ... return } ? It was already checked before that conditional statement so I replaced it with an unknown-message check: if (_sendComplete) return; if (msg==this_type) { do(that_thing); } else if (msg==that_type) do(the_other_thing); } else if (_sendComplete) { return; } But... even the first check is ultimately removed in r17004 to make _sendComplete *only* a signal from the blocking-send() thread to the _senderThread; eliminating a major race condition (if _senderThread was the first to notice that (1) client disconnected, or (2) prb aborted, all that the send() thread knew was "_sendComplete", when if it would just loop again it would notice the exact condition itself). It may be easier to read the final version of the transmitter / receiver, they are a lot more readable now, and I tested them with 10% packet loss and they worked fine. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080111/0182b014/attachment.htm From robert at emu.freenetproject.org Fri Jan 11 17:18:45 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 11:18:45 -0600 Subject: [freenet-dev] [freenet-cvs] r17015 - trunk/freenet/src/freenet/io/comm In-Reply-To: <20080111170823.585423A19DA@freenetproject.org> References: <20080111170823.585423A19DA@freenetproject.org> Message-ID: On Jan 11, 2008, at 11:08 AM, toad at freenetproject.org wrote: > Author: toad > Date: 2008-01-11 17:08:23 +0000 (Fri, 11 Jan 2008) > New Revision: 17015 > > Modified: > trunk/freenet/src/freenet/io/comm/MessageCore.java > Log: > delete ping handling Won't this break the simulator again? -- Robert Hailey From toad at amphibian.dyndns.org Fri Jan 11 17:41:32 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 17:41:32 +0000 Subject: [freenet-dev] [freenet-cvs] r17015 - trunk/freenet/src/freenet/io/comm In-Reply-To: References: <20080111170823.585423A19DA@freenetproject.org> Message-ID: <200801111741.32653.toad@amphibian.dyndns.org> On Friday 11 January 2008 17:18, Robert Hailey wrote: > > On Jan 11, 2008, at 11:08 AM, toad at freenetproject.org wrote: > > > Author: toad > > Date: 2008-01-11 17:08:23 +0000 (Fri, 11 Jan 2008) > > New Revision: 17015 > > > > Modified: > > trunk/freenet/src/freenet/io/comm/MessageCore.java > > Log: > > delete ping handling > > Won't this break the simulator again? Will it? I didn't see any references, but if you need it reinstate it. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/4fd5126c/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 17:40:31 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 17:40:31 +0000 Subject: [freenet-dev] Message priorities (and coalescing timeouts) was Re: [freenet-cvs] r16960 - trunk/freenet/src/freenet/node In-Reply-To: <47872A09.40109@cs.ucl.ac.uk> References: <20080107204520.4EAB139230A@freenetproject.org> <20080111000643.3ptzwun1mskc84gc@69.49.230.6> <47872A09.40109@cs.ucl.ac.uk> Message-ID: <200801111740.35908.toad@amphibian.dyndns.org> On Friday 11 January 2008 08:34, Michael Rogers wrote: > Robert Hailey wrote: > > Rejecting new > > requests would keep the data-transfer level traffic from being > > starved. > > Yup, that makes sense as long as it also applies to local requests. Which it does, as of some weeks back. BTW robert, we do have a limit on the number of running requests: the bandwidth liability limits. Now, is there any reason to prioritise different control messages differently? And do we want absolute priorities, or different timeouts (messages will be sent at t+100ms if they haven't been sent already at the moment, we could vary this by type), or both? For example: Group 0: 100ms coalescing timeout. Packet acknowledgments. These need to be sent fairly quickly. Group 1: 100ms coalescing timeout. Various messages with short timeouts, or which we need to send quickly for one reason or another. FNPAccepted FNPRejectedLoop FNPRejectedOverload FNPDataInsertRejected FNPSSKAccepted FNPCHKDataRequest FNPSSKDataRequest FNPRecentlyFailed (can be sent at Accepted time, or later) FNPInsertRequest FNPDataInsert FNPRejectedTimeout (counterpart of FNPDataInsert) FNPSSKInsertRequest FNPSSKAccepted FNPSSKPubKeyAccepted FNPOpennetAnnounceRequest FNPOpennetDisabled FNPOpennetNoderefRejected FNPPing/FNPPong - ???? FNPLinkPing/FNPLinkPong - ???? FNPProbeRequest - ???? FNPProbeRejected - ???? FNPDetectedIPAddress - good to have this arrive soon after connect FNPTime - ditto FNPSentPackets - ditto FNPDisconnect - good to have this arrive quickly as we may be shutting down UOMRequestRevocation UOMSendingRevocation FNPRoutingStatus FNPSwapRequest/Rejected/Reply/Commit/Complete/Rejected - 60 seconds timeout, but IMHO we need these to move fast Group 2: Messages which don't have particularly short timeouts. 200ms coalescing timeout? Or even more? allSent - not very important missingPacketNotification - no short timeout, however we do want it to arrive soonish so we can complete the transfer; maybe include in the next group??? allReceived - important message, 60 second timeout sendAborted - moderately important message, can go each way, 60 second timeout one way FNPBulkSendAborted - long timeout (5min) FNPBulkReceiveAborted - no timeout, dropped on completion FNPBulkReceivedAll - no timeout, dropped on completion nodeToNodeMessage - maybe treat this as bulk? FNPDataFound - long timeout (2min) FNPCHKDataFound - long timeout (2min) FNPRouteNotFound - long timeout (2min) FNPInsertReply - long timeout (2min) FNPSSKDataFound FNPDataInsertRejected - ?? FNPInsertTransfersCompleted - long timeout (2min) FNPOpennetCompletedAck - long timeout (2min) FNPOpennetConnectDestinationNew - long timeout (2min) FNPOpennetConnectReplyNew - long timeout (2min) FNPOpennetAnnounceReply - long timeout (4min) FNPOpennetAnnounceCompleted - long timeout (4min) FNPOpennetAnnounceNodeNotWanted - long timeout (4min) FNPOfferKey FNPProbeTrace - ???? FNPProbeReply - ???? FNPLocChangeNotification FNPRoutedPing/Pong FNPVoid UOMAnnounce UOMRequestMain UOMRequestExtra UOMSendingMain UOMSendingExtra Group 3: Bulk data transfer. 200ms coalescing timeout? Or even more? (note these are all around 1kB, they should all behave similarly to minimize the available data for traffic analysis) 200ms timeout? packetTransmit FNPBulkPacketSend FNPSSKInsertRequest FNPSSKDataFound FNPSSKPubKey When we do decide to send a packet, we should send as big a packet as possible, provided that this doesn't break the timeouts, so usually we would bundle some packet acks, some high priority messages which are past their timeout, some high priority messages which are not past their timeouts, and one big message. Another consideration: can delaying packets for coalescing cause us to grossly undershoot our bandwidth limit? > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/06d9a44f/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 17:43:57 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 17:43:57 +0000 Subject: [freenet-dev] [freenet-cvs] r16959 - trunk/freenet/src/freenet/node In-Reply-To: References: <20080107200730.C0B203A13AC@freenetproject.org> <200801102310.10329.toad@amphibian.dyndns.org> Message-ID: <200801111743.57548.toad@amphibian.dyndns.org> On Thursday 10 January 2008 23:35, Robert Hailey wrote: > > On Jan 10, 2008, at 5:10 PM, Matthew Toseland wrote: > > > With such an aggressive SEND_TIMEOUT, this borders on reviving > > NGRouting - > > it's a major change to routing, which since it hasn't been simulated > > will > > almost certainly cause major problems. No? > > > I'm thinking that the SEND_TIMEOUT will need to be increased, I really > am not familiar with what the average network condition will be like. > Feel free to change the SEND_TIMEOUTs to your liking. IMHO it should be equal to ACCEPTED_TIMEOUT. Although with message prioritisation I'm not sure conditionalSend does anything useful? > > Also, I noticed that the htl is still decremented if we skip a peer w/ > conditionalSend (or on disconnect). > > On Jan 10, 2008, at 5:20 PM, Matthew Toseland wrote: > > Just because our queue of data to send to a node is full doesn't > > mean its > > queue of data to send to us is full. Queue based load management > > would have > > to take the latter into account. In fact, it would have to take not > > only the > > data currently queued but also the data that is likely to become > > queued into > > account. The liability limiting code works on this principle. Various > > proposals for load management work similarly by passing back > > information on > > how many requests can be made in the near future - but these will > > not be > > deployed until they have been simulated. > > > > So please show me that this has no effect on routing, or revert it. > > My initial impression was that with SEND_TIMEOUT being either > ACCEPTED_TIMEOUT (equivalent to sendAsync) or one minute (equivalent > to sendSync) that it would be the same. It does change the behavior > though (as the packet is withdrawn from the send queue, or not sent). > Until it can be simulated, I can simply flip the send back to > sendSync(). > > -- > Robert Hailey > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/6f210b41/attachment.pgp From robert at emu.freenetproject.org Fri Jan 11 17:53:37 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 11:53:37 -0600 Subject: [freenet-dev] [freenet-cvs] r16959 - trunk/freenet/src/freenet/node In-Reply-To: <200801111743.57548.toad@amphibian.dyndns.org> References: <20080107200730.C0B203A13AC@freenetproject.org> <200801102310.10329.toad@amphibian.dyndns.org> <200801111743.57548.toad@amphibian.dyndns.org> Message-ID: <74912B7C-73C2-4F3A-A2C5-38582B84AB4A@emu.freenetproject.org> On Jan 11, 2008, at 11:43 AM, Matthew Toseland wrote: >> I'm thinking that the SEND_TIMEOUT will need to be increased, I >> really >> am not familiar with what the average network condition will be like. >> Feel free to change the SEND_TIMEOUTs to your liking. > > IMHO it should be equal to ACCEPTED_TIMEOUT. Although with message > prioritisation I'm not sure conditionalSend does anything useful? I agree, and have reverted r16959 in r17006. With message prioritization, the marginal utility it *might* provide in extreme cases will both (1) be quickly detected with the standard backoff/routing, and (2) not be worth the increased code complexity. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080111/c1751b14/attachment.htm From robert at emu.freenetproject.org Fri Jan 11 17:59:15 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 11:59:15 -0600 Subject: [freenet-dev] Message priorities (and coalescing timeouts) In-Reply-To: <200801111740.35908.toad@amphibian.dyndns.org> References: <20080107204520.4EAB139230A@freenetproject.org> <20080111000643.3ptzwun1mskc84gc@69.49.230.6> <47872A09.40109@cs.ucl.ac.uk> <200801111740.35908.toad@amphibian.dyndns.org> Message-ID: On Jan 11, 2008, at 11:40 AM, Matthew Toseland wrote: > FNPRecentlyFailed (can be sent at Accepted time, or later) As best I can tell, the RecentlyFailed code is totally dead. Right now a node will relay them in RequestSender (or Handler?), but will never start one. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080111/5bcb48f6/attachment.htm From robert at emu.freenetproject.org Fri Jan 11 18:10:12 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 12:10:12 -0600 Subject: [freenet-dev] Message priorities (and coalescing timeouts) In-Reply-To: <200801111740.35908.toad@amphibian.dyndns.org> References: <20080107204520.4EAB139230A@freenetproject.org> <20080111000643.3ptzwun1mskc84gc@69.49.230.6> <47872A09.40109@cs.ucl.ac.uk> <200801111740.35908.toad@amphibian.dyndns.org> Message-ID: <66392E4E-7279-49C4-B16B-FB7BCFCE6EA8@emu.freenetproject.org> On Jan 11, 2008, at 11:40 AM, Matthew Toseland wrote: > On Friday 11 January 2008 08:34, Michael Rogers wrote: >> Robert Hailey wrote: >>> Rejecting new >>> requests would keep the data-transfer level traffic from being >>> starved. >> >> Yup, that makes sense as long as it also applies to local requests. > > Which it does, as of some weeks back. > > BTW robert, we do have a limit on the number of running requests: the > bandwidth liability limits. > > Now, is there any reason to prioritise different control messages > differently? > And do we want absolute priorities, or different timeouts (messages > will be > sent at t+100ms if they haven't been sent already at the moment, we > could > vary this by type), or both? > > For example: > > Group 0: > 100ms coalescing timeout. > Packet acknowledgments. > These need to be sent fairly quickly. > > Group 1: > 100ms coalescing timeout. > Various messages with short timeouts, or which we need to send > quickly for one > reason or another. > FNPAccepted > FNPRejectedLoop > FNPRejectedOverload > FNPDataInsertRejected > FNPSSKAccepted > FNPCHKDataRequest > FNPSSKDataRequest > FNPRecentlyFailed (can be sent at Accepted time, or later) > FNPInsertRequest > FNPDataInsert > FNPRejectedTimeout (counterpart of FNPDataInsert) > FNPSSKInsertRequest > FNPSSKAccepted > FNPSSKPubKeyAccepted > FNPOpennetAnnounceRequest > FNPOpennetDisabled > FNPOpennetNoderefRejected > FNPPing/FNPPong - ???? > FNPLinkPing/FNPLinkPong - ???? > FNPProbeRequest - ???? > FNPProbeRejected - ???? > FNPDetectedIPAddress - good to have this arrive soon after connect > FNPTime - ditto > FNPSentPackets - ditto > FNPDisconnect - good to have this arrive quickly as we may be > shutting down > UOMRequestRevocation > UOMSendingRevocation > FNPRoutingStatus > FNPSwapRequest/Rejected/Reply/Commit/Complete/Rejected - 60 seconds > timeout, > but IMHO we need these to move fast > > Group 2: > Messages which don't have particularly short timeouts. > 200ms coalescing timeout? Or even more? > allSent - not very important > missingPacketNotification - no short timeout, however we do want it > to arrive > soonish so we can complete the transfer; maybe include in the next > group??? > allReceived - important message, 60 second timeout > sendAborted - moderately important message, can go each way, 60 > second timeout > one way > FNPBulkSendAborted - long timeout (5min) > FNPBulkReceiveAborted - no timeout, dropped on completion > FNPBulkReceivedAll - no timeout, dropped on completion > nodeToNodeMessage - maybe treat this as bulk? > FNPDataFound - long timeout (2min) > FNPCHKDataFound - long timeout (2min) > FNPRouteNotFound - long timeout (2min) > FNPInsertReply - long timeout (2min) > FNPSSKDataFound > FNPDataInsertRejected - ?? > FNPInsertTransfersCompleted - long timeout (2min) > FNPOpennetCompletedAck - long timeout (2min) > FNPOpennetConnectDestinationNew - long timeout (2min) > FNPOpennetConnectReplyNew - long timeout (2min) > FNPOpennetAnnounceReply - long timeout (4min) > FNPOpennetAnnounceCompleted - long timeout (4min) > FNPOpennetAnnounceNodeNotWanted - long timeout (4min) > FNPOfferKey > FNPProbeTrace - ???? > FNPProbeReply - ???? > FNPLocChangeNotification > FNPRoutedPing/Pong > FNPVoid > UOMAnnounce > UOMRequestMain > UOMRequestExtra > UOMSendingMain > UOMSendingExtra > > Group 3: > Bulk data transfer. > 200ms coalescing timeout? Or even more? > (note these are all around 1kB, they should all behave similarly to > minimize > the available data for traffic analysis) > 200ms timeout? > packetTransmit > FNPBulkPacketSend > FNPSSKInsertRequest > FNPSSKDataFound > FNPSSKPubKey > > When we do decide to send a packet, we should send as big a packet as > possible, provided that this doesn't break the timeouts, so usually > we would > bundle some packet acks, some high priority messages which are past > their > timeout, some high priority messages which are not past their > timeouts, and > one big message. > > Another consideration: can delaying packets for coalescing cause us > to grossly > undershoot our bandwidth limit? I like this schedule of groupings, but for lack of a point of reference I can't speak towards the coalescing delays. What do you mean by delay causing the bandwidth limit to be undershot? It seems to me that if (for example) the packetTransmitters use the same master throttle, then packets getting ahead of them would cause a trivial creep of the send queue, but that's it. -- Robert Hailey From toad at amphibian.dyndns.org Fri Jan 11 18:10:32 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 18:10:32 +0000 Subject: [freenet-dev] Message priorities (and coalescing timeouts) In-Reply-To: References: <20080107204520.4EAB139230A@freenetproject.org> <200801111740.35908.toad@amphibian.dyndns.org> Message-ID: <200801111810.38596.toad@amphibian.dyndns.org> On Friday 11 January 2008 17:59, Robert Hailey wrote: > > On Jan 11, 2008, at 11:40 AM, Matthew Toseland wrote: > > > FNPRecentlyFailed (can be sent at Accepted time, or later) > > As best I can tell, the RecentlyFailed code is totally dead. Right now > a node will relay them in RequestSender (or Handler?), but will never > start one. Yeah, it's part of the half-finished support for Ultra Lightweight Passive Requests. I will finish it after we have fully working opennet (i.e. opennet harvesting). It is essential for the next generation of Frost and is more widely helpful. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/996e42e8/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 18:18:58 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 18:18:58 +0000 Subject: [freenet-dev] Message priorities (and coalescing timeouts) In-Reply-To: <66392E4E-7279-49C4-B16B-FB7BCFCE6EA8@emu.freenetproject.org> References: <20080107204520.4EAB139230A@freenetproject.org> <200801111740.35908.toad@amphibian.dyndns.org> <66392E4E-7279-49C4-B16B-FB7BCFCE6EA8@emu.freenetproject.org> Message-ID: <200801111818.59091.toad@amphibian.dyndns.org> On Friday 11 January 2008 18:10, Robert Hailey wrote: > > On Jan 11, 2008, at 11:40 AM, Matthew Toseland wrote: > > > On Friday 11 January 2008 08:34, Michael Rogers wrote: > >> Robert Hailey wrote: > >>> Rejecting new > >>> requests would keep the data-transfer level traffic from being > >>> starved. > >> > >> Yup, that makes sense as long as it also applies to local requests. > > > > Which it does, as of some weeks back. > > > > BTW robert, we do have a limit on the number of running requests: the > > bandwidth liability limits. > > > > Now, is there any reason to prioritise different control messages > > differently? > > And do we want absolute priorities, or different timeouts (messages > > will be > > sent at t+100ms if they haven't been sent already at the moment, we > > could > > vary this by type), or both? > > > > For example: > > > > Group 0: > > 100ms coalescing timeout. > > Packet acknowledgments. > > These need to be sent fairly quickly. > > > > Group 1: > > 100ms coalescing timeout. > > Various messages with short timeouts, or which we need to send > > quickly for one > > reason or another. > > FNPAccepted > > FNPRejectedLoop > > FNPRejectedOverload > > FNPDataInsertRejected > > FNPSSKAccepted > > FNPCHKDataRequest > > FNPSSKDataRequest > > FNPRecentlyFailed (can be sent at Accepted time, or later) > > FNPInsertRequest > > FNPDataInsert > > FNPRejectedTimeout (counterpart of FNPDataInsert) > > FNPSSKInsertRequest > > FNPSSKAccepted > > FNPSSKPubKeyAccepted > > FNPOpennetAnnounceRequest > > FNPOpennetDisabled > > FNPOpennetNoderefRejected > > FNPPing/FNPPong - ???? > > FNPLinkPing/FNPLinkPong - ???? > > FNPProbeRequest - ???? > > FNPProbeRejected - ???? > > FNPDetectedIPAddress - good to have this arrive soon after connect > > FNPTime - ditto > > FNPSentPackets - ditto > > FNPDisconnect - good to have this arrive quickly as we may be > > shutting down > > UOMRequestRevocation > > UOMSendingRevocation > > FNPRoutingStatus > > FNPSwapRequest/Rejected/Reply/Commit/Complete/Rejected - 60 seconds > > timeout, > > but IMHO we need these to move fast > > > > Group 2: > > Messages which don't have particularly short timeouts. > > 200ms coalescing timeout? Or even more? > > allSent - not very important > > missingPacketNotification - no short timeout, however we do want it > > to arrive > > soonish so we can complete the transfer; maybe include in the next > > group??? > > allReceived - important message, 60 second timeout > > sendAborted - moderately important message, can go each way, 60 > > second timeout > > one way > > FNPBulkSendAborted - long timeout (5min) > > FNPBulkReceiveAborted - no timeout, dropped on completion > > FNPBulkReceivedAll - no timeout, dropped on completion > > nodeToNodeMessage - maybe treat this as bulk? > > FNPDataFound - long timeout (2min) > > FNPCHKDataFound - long timeout (2min) > > FNPRouteNotFound - long timeout (2min) > > FNPInsertReply - long timeout (2min) > > FNPSSKDataFound > > FNPDataInsertRejected - ?? > > FNPInsertTransfersCompleted - long timeout (2min) > > FNPOpennetCompletedAck - long timeout (2min) > > FNPOpennetConnectDestinationNew - long timeout (2min) > > FNPOpennetConnectReplyNew - long timeout (2min) > > FNPOpennetAnnounceReply - long timeout (4min) > > FNPOpennetAnnounceCompleted - long timeout (4min) > > FNPOpennetAnnounceNodeNotWanted - long timeout (4min) > > FNPOfferKey > > FNPProbeTrace - ???? > > FNPProbeReply - ???? > > FNPLocChangeNotification > > FNPRoutedPing/Pong > > FNPVoid > > UOMAnnounce > > UOMRequestMain > > UOMRequestExtra > > UOMSendingMain > > UOMSendingExtra > > > > Group 3: > > Bulk data transfer. > > 200ms coalescing timeout? Or even more? > > (note these are all around 1kB, they should all behave similarly to > > minimize > > the available data for traffic analysis) > > 200ms timeout? > > packetTransmit > > FNPBulkPacketSend > > FNPSSKInsertRequest > > FNPSSKDataFound > > FNPSSKPubKey > > > > When we do decide to send a packet, we should send as big a packet as > > possible, provided that this doesn't break the timeouts, so usually > > we would > > bundle some packet acks, some high priority messages which are past > > their > > timeout, some high priority messages which are not past their > > timeouts, and > > one big message. > > > > Another consideration: can delaying packets for coalescing cause us > > to grossly > > undershoot our bandwidth limit? > > I like this schedule of groupings, but for lack of a point of > reference I can't speak towards the coalescing delays. Well, at the moment we have a maximum coalescing delay of 100ms, so no message will wait much more than 100ms to be sent, modulo bandwidth limits and packet throttle. We also send messages when we have more than [ MTU - 100 bytes ] worth of data to send. > > What do you mean by delay causing the bandwidth limit to be undershot? > It seems to me that if (for example) the packetTransmitters use the > same master throttle, then packets getting ahead of them would cause a > trivial creep of the send queue, but that's it. I mean can the coalescing delay reduce our bandwidth utilisation? If one connection is maxed out, then no, not for that connection, because there will constantly be packets to send, but if we have data trickling out on various connections, and we always wait 200ms before sending it ... I suppose that's also trivial, because as soon as we have another packet to send we'll send the first one. Okay, you're right, there's no problem here. Are you interested in implementing message priorities? MessageItem and PacketSender are the most relevant classes. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/d2b10b3b/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 18:22:58 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 18:22:58 +0000 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <690CA447-2388-449B-8030-BE7F56BF6C6D@emu.freenetproject.org> References: <20080104182211.0332D390A59@freenetproject.org> <200801082133.51052.toad@amphibian.dyndns.org> <690CA447-2388-449B-8030-BE7F56BF6C6D@emu.freenetproject.org> Message-ID: <200801111822.59274.toad@amphibian.dyndns.org> On Tuesday 08 January 2008 22:57, Robert Hailey wrote: > > On Jan 8, 2008, at 3:33 PM, Matthew Toseland wrote: > > > On Saturday 05 January 2008 00:50, Robert Hailey wrote: > >> > >> Interestingly (now that I have got the simulator running), this > >> 'general timeout' appears even in simulations between nodes on the > >> same machine. Unless I coded something wrong, perhaps there is an > >> added delay or missing response somewhere which is not obvious? > > > > Entirely possible. Fixing it would be better than an arbitrary > > cutoff when we > > are still able to potentially find the data, and still have enough > > HTL to do > > so. > > On Jan 8, 2008, at 2:27 PM, Matthew Toseland wrote: > > On Friday 04 January 2008 18:32, Robert Hailey wrote: > >> > >> Apparently until this revision 16886, (so long as any one node does > >> not timeout) a node will take as long as necessary to exhaust > >> routable > >> peers. Even long after the original requestor has given up on that > >> node. > > > > Is there any evidence that this happens in practice? Surely the HTL > > should > > prevent excessive searching in most cases? > > There is, in fact. The timeout itself (which I have been running on my > node for a while) is evidence of the behavior (which to be seems > incorrect). > > Jan 08, 2008 20:03:41:146 (freenet.node.RequestSender, RequestSender > for UID -3998139406700477577, ERROR): discontinuing non-local request > search, general timeout (6 attempts, 3 overloads) > ... > Jan 08, 2008 20:12:21:226 (freenet.node.RequestSender, RequestSender > for UID 60170596711015291, ERROR): discontinuing non-local request > search, general timeout (1 attempts, 3 overloads) Ouch. How common is this? > > You see... in the first log statement the node tried six peers before > running out of time. In the second case (which occurs quite > frequently), the node used the entire 2 minutes waiting on a response > from one node (FETCH_TIMEOUT); if it were allowed to continue to the > next node it could (65%) spend another 2 minutes on just-that-node. > > >> To the best of my knowledge, all of the upstream nodes will not > >> respond with the LOOP rejection before then. And even well before the > >> worst case, this effect can accrue across many nodes in the path. > > > > If the same request is routed to a node which is already running it, > > it > > rejects it with RejectedLoop. If it's routed to a node which has > > recently ran > > it, it again rejects it. If it is a different request for the same > > key, it > > may be coalesced. > > If you mean that the RECENTLY_FAILED mechanism would keep this in > check... I see this idea in many places, but I cannot see where this > is actually implemented. The only place I see that makes a > FNPRecentlyFailed message is in RequestHandler (upon it's > RequestSender having received one). It's part of the unfinished ULPRs system. It will be implemented after I have opennet fully sorted out. > > Presently the node will "RejectLoop" if it is one of the last 10000 > completed requests. My node runs through that many requests in about > 16 minutes. With this logging statement it is already shown that a > request can last longer than 2 minutes for one peer (and most nodes > have 20). If you assume that a request takes 4 minutes (two peers, > VERY optimistic), then it would then only take 4 nodes ('near' each > other) to generate a request-live-lock (absent the HTL; the request > would never drop from the network); each trying two of it's other > peers, and then the next in the 4-chain. Okay, this is a problem. > > I do not think that this timeout I added is arbitrary. As I understand > Ian's original networking theory, a request is not valid after the > originator has timed out. In much the same way that a single node > fatally timing out collapses the request chain, so too should a node > 'taking too long' (as that node *IS* the one fatally timing out the > chain). Well, we can't easily inform downstream nodes of a request timing out. And we can't include the updated timeout on each hop either (for security reasons). > > But on the other hand, I do understand your point about the HTL, and > that it would keep the request from continuing indefinitely; it seems > like it could be quite a waste of network resources. Certainly beyond > that point in time (where the requester has fatally timed out) no > response should be sent back to the source (that could be many of the > unclaimed fifo packets); or maybe just if the data is finally found > (you mentioned ULPRs). Maybe there is another reason for this behaviour. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/42e5eee8/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 18:24:34 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 18:24:34 +0000 Subject: [freenet-dev] htl problem was r16886 In-Reply-To: References: <20080104182211.0332D390A59@freenetproject.org> <200801082039.43208.toad@amphibian.dyndns.org> Message-ID: <200801111824.34762.toad@amphibian.dyndns.org> On Wednesday 09 January 2008 19:59, Robert Hailey wrote: > > On Jan 8, 2008, at 2:39 PM, Matthew Toseland wrote: > > >> In either case, resuming a request after we know that the upstream > >> peer has forgotten about it could be very bad. Assuming 20 peers (ala > >> opennet), the theoretical worst-case-per-node is that the last new > >> request will leave the node about 40 minutes from when it entered the > >> node. > > > > Not possible because of HTL. We will DNF (or timeout) before we've > > visited all > > 20 nodes. We don't allow RNFs to increase the HTL or change the > > nearest-loc-found, but we do allow them to decrease the HTL. And we > > decrement > > the HTL unless nearestLoc improves (the HTL decrement/reset code > > really needs > > to be looked at, it's far from clear and may be wrong in places). > > One caveat: > > at HTL=10 or HTL=1, whether or not the node decrements is determined > > probabilistically (the decision is made once per source node). > > It looks like the htl is decremented with respect to the source of the > node (not the node we are routing to), so at 10 or 1 if the request > came from a node w/o the probabalitic decrement (or if prob. decrement > is disabled!) then the standard request search will try all it's peers. Ah. Yes, you are right. On the other hand, each of the nodes we route to and get an RNF from should itself reduce the HTL, in almost all cases, so I'm not sure that this fully explains it. Anyway, fix it - decrement with respect to the node we are routing to after an RNF. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/7a367980/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 18:42:58 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 18:42:58 +0000 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <0CC1E253-3C9F-483A-8137-D43B74F2E86B@emu.freenetproject.org> References: <20080104182211.0332D390A59@freenetproject.org> <200801082027.32705.toad@amphibian.dyndns.org> <0CC1E253-3C9F-483A-8137-D43B74F2E86B@emu.freenetproject.org> Message-ID: <200801111843.03060.toad@amphibian.dyndns.org> On Wednesday 09 January 2008 17:14, Robert Hailey wrote: > > On Jan 8, 2008, at 2:27 PM, Matthew Toseland wrote: > > > On Friday 04 January 2008 18:32, Robert Hailey wrote: > >> > >> Apparently until this revision 16886, (so long as any one node does > >> not timeout) a node will take as long as necessary to exhaust > >> routable > >> peers. Even long after the original requestor has given up on that > >> node. > > > > Is there any evidence that this happens in practice? Surely the HTL > > should > > prevent excessive searching in most cases? > > I have reverted r16886, as it appears to be based on a misunderstand > of how requests work against the topology of the network (r16980). I was rather hoping to be talked into accepting 16886 ... we should at least have some logging in such cases IMHO. Requests really shouldn't be taking that long - maybe it's related to the HTL problem, maybe we have such a perverse network topology that we are resetting HTL time after time after time, I dunno, simulations would be interesting. > > Instead of canceling the search request, I have put similar logic in > RequestHandler to simply not respond to very old requests (r16982), Ok. Maybe this is where we should have such logging. > and not to hang onto the thread if no response is therefore necessary > (r16983). > > r16983, (given the timeout) makes extra calls to rejectedOverload(); I > imagine that at the timeout (every 2 minutes while the local request > is running) they are benign, but I can code up an infinite wait (as > before) if needed. Well it's not strictly infinite - it's 2min*peers at most. You reverted this in 16985. > > I'm going to try and stay away from high level changes for a while, > but at your suggestion I will look at the HTL code. You are more than welcome to continue to investigate the timeouts, they have been a problem for a long time. Do they still show up in simulation with the HTL problem fixed? I'm very happy to have somebody else working on core stuff, I'm equally happy to point out any obvious problems with their commits. :) -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/84cca761/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 18:45:27 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 18:45:27 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <477E7E1D.9060300@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801041410.22489.toad@amphibian.dyndns.org> <477E7E1D.9060300@cs.ucl.ac.uk> Message-ID: <200801111845.27956.toad@amphibian.dyndns.org> On Friday 04 January 2008 18:42, Michael Rogers wrote: > Matthew Toseland wrote: > > I was thinking of long-term requests in general - large downloads for example. > > > > Do we need to do this on the return journey or only on the outward? > > Both, otherwise the nodes in the return tunnel can gather successor samples. Because the next-node is more likely than any other to be the originator. Right. > > >> Sounds promising - but could that result in some nodes not being allowed > >> to join any cell? > > > > Yes. If you're a leaf node you can't use premix routing. > > Not just leaves, but perhaps nodes whose neighbours all belong to > different clusters? Yeah, this could be a problem... Dunno how we'd deal with that... > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/8dd9ce48/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 18:48:28 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 18:48:28 +0000 Subject: [freenet-dev] Predecessor sampling and tunnel padding was Re: One tunnel per request: justification In-Reply-To: <477E8111.2010703@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801041416.54969.toad@amphibian.dyndns.org> <477E8111.2010703@cs.ucl.ac.uk> Message-ID: <200801111848.28207.toad@amphibian.dyndns.org> On Friday 04 January 2008 18:55, Michael Rogers wrote: > Matthew Toseland wrote: > > Opennet nodes won't participate in premix routing. > > What happens if the darknet nodes don't form a connected subnetwork? Presumably it will only work in darknet subnetworks. > > > Okay, so on the one hand, more requests per tunnel means easier tagging > > (noisier samples). On the other hand, less requests per tunnel means more > > tunnels and more predecessor samples (more samples). Hmm... I suppose finding > > the optimal number of requests per tunnel is a matter for simulation? Would > > it be hideously complicated? > > As always, the problem is making it realistic... we can do a simplified > simulation but it might not capture some relevant feature (like > daily/weekly uptime patterns, just as an example of something that's > likely to be relevant but hard to simulate realistically). :| > > > Well yeah but Frost IDs and splitfiles, and splitfiles and splitfiles, often > > are linked. So it's not easy. > > Absolutely - we'd probably need to rely on clients to make informed > choices (eg allow an arbitrary "batch ID" to be associated with each > request, and try to send requests with the same batch ID through the > same tunnel or set of tunnels, and requests with different batch IDs > through different tunnels). > > >> If the routes are random, two tunnels that pass through different > >> neighbours of X don't reveal any more information than two tunnels that > >> pass through the same neighbour of X: the probability of the tunnels > >> parting ways at X doesn't depend on whether X is the initiator. > > > > It doesn't? I had assumed they always parted ways at the beginning... hmmm. > > Can you elaborate a little? > > If each hop of each tunnel is chosen independently and randomly, then > once two tunnels reach X, the probability that their next hop is the > same doesn't depend on the route so far - so it's the same as the > probability of two tunnels that started at X having the same next hop. But it's fairly unlikely that two tunnels from Y both reached X, isn't it? > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/5e8617ed/attachment.pgp From m.rogers at cs.ucl.ac.uk Fri Jan 11 19:03:40 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 11 Jan 2008 19:03:40 +0000 Subject: [freenet-dev] Predecessor sampling and tunnel padding was Re: One tunnel per request: justification In-Reply-To: <200801111848.28207.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801041416.54969.toad@amphibian.dyndns.org> <477E8111.2010703@cs.ucl.ac.uk> <200801111848.28207.toad@amphibian.dyndns.org> Message-ID: <4787BD8C.2030809@cs.ucl.ac.uk> Matthew Toseland wrote: >> What happens if the darknet nodes don't form a connected subnetwork? > > Presumably it will only work in darknet subnetworks. So the cell is limited by the size of the darknet subnetwork? Makes sense, but if a darknet subnetwork generally corresponds to a group or organisation then remaining anonymous within that group might not as useful as remaining anonymous within a group of strangers. > But it's fairly unlikely that two tunnels from Y both reached X, isn't it? Right, but the point I was trying to make (which really isn't very important) is that for the purposes of determining whether X is the initiator, it's irrelevant whether the attacker sees two tunnels exiting X by different paths or two tunnels exiting X by the same path. I'm not trying to say that two tunnels exiting X don't tell the attacker anything - they do. But it's irrelevant where the tunnels go after exiting X, because the routes are random. Cheers, Michael From m.rogers at cs.ucl.ac.uk Fri Jan 11 19:06:03 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 11 Jan 2008 19:06:03 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <200801111845.27956.toad@amphibian.dyndns.org> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801041410.22489.toad@amphibian.dyndns.org> <477E7E1D.9060300@cs.ucl.ac.uk> <200801111845.27956.toad@amphibian.dyndns.org> Message-ID: <4787BE1B.8090508@cs.ucl.ac.uk> Matthew Toseland wrote: >> Not just leaves, but perhaps nodes whose neighbours all belong to >> different clusters? > > Yeah, this could be a problem... Dunno how we'd deal with that... I guess if opennet nodes are going to be excluded from premix routing then some darknet nodes could also be excluded. I have some notes somewhere on various ways of defining cluster and cliques... I'll see if there's anything that could be used to agree on the membership of a cell. Cheers, Michael From toad at amphibian.dyndns.org Fri Jan 11 19:10:21 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 19:10:21 +0000 Subject: [freenet-dev] Tunnels vs premix routing was Re: Vulnerability in inserting the manifest before finishing In-Reply-To: <477F1280.40107@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801042039.50495.toad@amphibian.dyndns.org> <477F1280.40107@cs.ucl.ac.uk> Message-ID: <200801111910.26687.toad@amphibian.dyndns.org> On Saturday 05 January 2008 05:15, Michael Rogers wrote: > Matthew Toseland wrote: > > The 50% chance of decrementing > > at 10 means that 3 resets equals 6 nodes with HTL 10... > > It's more complicated than that... if the HTL is 10 and the closest > location isn't the previous hop's location, then the attacker knows the > previous hop doesn't decrement at 10. It's a function of the input node, not the output node. Although that's problematic on rerouting, so we may change it to be a function of the input node for the first node tried, and then a function of the output node... Hmmm. Why did we want to choose it once per node again? I'm sure there was a good reason, but I don't remember it... > Likewise if the HTL is 9 and the > closest location isn't the previous hop's location, the previous hop > decrements at 10. Either way, the attacker can use that knowledge for > the rest of the session. Okay, so for a single request, if HTL is 10, and closestLocation is equal to the node's location, the node could be the originator or it could have reset it. If HTL is 10, and closestLocation is not equal to the node's location, the node cannot be the originator and cannot have reset the HTL. So without keeping any state, we just look for requests which have closestLocation equal to the previous node's location, and we have a 1 in (# resets per request + 1) chance that it's the originator (1 in 4 on the previous assumption). Ouch! I'm not sure how the additional state we could keep from the above would help the attacker. For a non-resetting HTL, the odds are even worse, unless we have a long probabilistic htl=max stage. So plainly a weighted coin would help significantly. The problem is that a weighted coin would increase the variance in path length *significantly*, cause more no-fault timeouts, cause more no-fault failures (too short a path), make request coalescing extremely difficult, and likewise with ULPRs. Is there an alternative? Tunnels, on the other hand, could avoid a lot of these problems. Once the request is out of the tunnel, it can be happily coalesced etc. And for very popular keys, ULPRs will work well even with tunnels; for less popular keys obviously what you gain in anonymity you lose in reliability. > > > When an attacker gets a tunneled request, he knows 1) there is a 20% chance > > that the node is the originator, and 2) that he is close to the originator, > > because he is early on the route: the tunnel is only active at the beginning. > > Longer tunnels would help in both cases (at a cost, of course). Ideally > the tunnels would be long enough to reach any node with equal > probability, but maybe that's not realistic... > > Let's say we need an average of 10 hops. A weighted coin gives an > exponential hop count distribution (actually geometric but close > enough), so 25% of tunnels will have <= 2.9 hops, 50% will have <= 6.9 > hops, 75% will have <= 13.9 hops. Likewise the attacker will know > there's a 25% chance the initiator is within 2.9 hops, etc. (Not sure > how to handle the rounding here but you get the picture.) And a 10% chance of being directly connected. Right. 10 hops is quite expensive... do we want to have it customisable-per-request? Paranoid requestors would then stick out... OTOH this might be best, 99% of people will use the default setting. > > > For a regular request, the HTL can be 10 much later on in the request. In > > which case the predecessor sample is useless, and the key is likely close to > > the location of the attacker. > > Good point about the distance - can the attacker give less weight to > samples where the requested key is close to the attacker, on the > assumption that they're more likely to have travelled several hops already? Seems plausible. > > > Somewhere on this thread you said that the probability of two related tunnels > > coming from the same node is independant of that node being the originator. I > > don't understand that. > > I didn't mean that exactly - the fact that both tunnels pass through X > *does* suggest that X is the initiator, I'm just saying it's irrelevant > where they go after leaving X. An attacker doesn't learn any more by > linking two tunnels that pass from X to different neighbours than he > learns by linking two tunnels that pass from X to the same neighbour, > because the paths of the tunnels are independent, so the fact that they > did or didn't part company at X doesn't tell the attacker anything about > where they originated. It was a minor point really. Ok, this is true. How much information does an attacker gain from linking two tunnels from the same X ? A fairly high confidence that X is the originator, surely? Higher than the 10% or 20% we're talking about above...? -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/dc0bf15e/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 19:11:25 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 19:11:25 +0000 Subject: [freenet-dev] Predecessor sampling and tunnel padding was Re: One tunnel per request: justification In-Reply-To: <4787BD8C.2030809@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801111848.28207.toad@amphibian.dyndns.org> <4787BD8C.2030809@cs.ucl.ac.uk> Message-ID: <200801111911.26127.toad@amphibian.dyndns.org> On Friday 11 January 2008 19:03, Michael Rogers wrote: > Matthew Toseland wrote: > >> What happens if the darknet nodes don't form a connected subnetwork? > > > > Presumably it will only work in darknet subnetworks. > > So the cell is limited by the size of the darknet subnetwork? Makes > sense, but if a darknet subnetwork generally corresponds to a group or > organisation then remaining anonymous within that group might not as > useful as remaining anonymous within a group of strangers. True enough. Opennet sucks, we need to build the darknet, this has been a parameter of my work on 0.7 since the beginning. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/54f05083/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 19:12:24 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 19:12:24 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <4787BE1B.8090508@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801111845.27956.toad@amphibian.dyndns.org> <4787BE1B.8090508@cs.ucl.ac.uk> Message-ID: <200801111912.24377.toad@amphibian.dyndns.org> On Friday 11 January 2008 19:06, Michael Rogers wrote: > Matthew Toseland wrote: > >> Not just leaves, but perhaps nodes whose neighbours all belong to > >> different clusters? > > > > Yeah, this could be a problem... Dunno how we'd deal with that... > > I guess if opennet nodes are going to be excluded from premix routing > then some darknet nodes could also be excluded. Opennet nodes might have a completely different premix routing mechanism, something closer to I2P (more opportunity for cheating at the peer selection layer). > > I have some notes somewhere on various ways of defining cluster and > cliques... I'll see if there's anything that could be used to agree on > the membership of a cell. That would be useful. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/bdffc53f/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 19:33:44 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 19:33:44 +0000 Subject: [freenet-dev] [freenet-cvs] r16990 - trunk/freenet/src/freenet/io/xfer In-Reply-To: <20080110003314.3570147BC37@freenetproject.org> References: <20080110003314.3570147BC37@freenetproject.org> Message-ID: <200801111933.50103.toad@amphibian.dyndns.org> On Thursday 10 January 2008 00:33, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-10 00:33:13 +0000 (Thu, 10 Jan 2008) > New Revision: 16990 > > Modified: > trunk/freenet/src/freenet/io/xfer/BlockReceiver.java > Log: > tell the transmitter if we are not listening anymore, and why Interesting. This is because we register senders, so we can tell them that we aborted, but we don't register receivers, so abort() can't tell them. I have wondered about dumping Block* in favour of Bulk* in the next iteration of the low-level code - we don't need most of Block*'s complexity, since the next layer down provides reliable packet delivery anyway. > > Modified: trunk/freenet/src/freenet/io/xfer/BlockReceiver.java > =================================================================== > --- trunk/freenet/src/freenet/io/xfer/BlockReceiver.java 2008-01-10 00:15:37 UTC (rev 16989) > +++ trunk/freenet/src/freenet/io/xfer/BlockReceiver.java 2008-01-10 00:33:13 UTC (rev 16990) > @@ -51,6 +51,7 @@ > /** packet : Integer -> reportTime : Long * */ > HashMap _recentlyReportedMissingPackets = new HashMap(); > ByteCounter _ctr; > + boolean sentAborted; > > public BlockReceiver(MessageCore usm, PeerContext sender, long uid, PartiallyReceivedBlock prb, ByteCounter ctr) { > _sender = sender; > @@ -62,6 +63,7 @@ > > public void sendAborted(int reason, String desc) throws NotConnectedException { > _usm.send(_sender, DMT.createSendAborted(_uid, reason, desc), _ctr); > + sentAborted=true; > } > > public byte[] receive() throws RetrievalException { > @@ -153,6 +155,14 @@ > // We didn't cause it?! > Logger.error(this, "Caught in receive - probably a bug as receive sets it: "+e); > throw new RetrievalException(RetrievalException.UNKNOWN, "Aborted?"); > + } finally { > + try { > + if (_prb.isAborted() && !sentAborted) { > + sendAborted(_prb.getAbortReason(), _prb.getAbortDescription()); > + } > + } catch (NotConnectedException e) { > + //ignore > + } > } > } > } > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/197f75b3/attachment.pgp From toad at amphibian.dyndns.org Fri Jan 11 19:37:19 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 19:37:19 +0000 Subject: [freenet-dev] [freenet-cvs] r16940 - trunk/freenet/src/freenet/l10n In-Reply-To: <20080106130844.6351147B849@freenetproject.org> References: <20080106130844.6351147B849@freenetproject.org> Message-ID: <200801111937.19777.toad@amphibian.dyndns.org> On Sunday 06 January 2008 13:08, you wrote: > Author: nextgens > Date: 2008-01-06 13:08:43 +0000 (Sun, 06 Jan 2008) > New Revision: 16940 > > Modified: > trunk/freenet/src/freenet/l10n/freenet.l10n.de.properties > trunk/freenet/src/freenet/l10n/freenet.l10n.es.properties > trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties > Log: > l10n: sed -i '/#freenet/d' *properties ... > > it\'s better to have an incomplete translation than an outdated one Are these going to be updated? Do we have maintainers for these languages? > > Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.de.properties > =================================================================== > --- trunk/freenet/src/freenet/l10n/freenet.l10n.de.properties 2008-01-06 13:04:12 UTC (rev 16939) > +++ trunk/freenet/src/freenet/l10n/freenet.l10n.de.properties 2008-01-06 13:08:43 UTC (rev 16940) > @@ -696,8 +696,6 @@ > PeerManagerUserAlert.connErrorTitle=Einige Partner k?nnen sich nicht verbinden > PeerManagerUserAlert.noConns=Dieser Knoten war bis jetzt nicht in der Lage, sich mit anderen Knoten zu verbinden; er wird nicht normal funktionieren. Hoffentlich werden sich ein paar Ihrer Partner bald verbinden; wenn nicht, versuchen Sie einige weitere Partner zu finden. Sie brauchen zu jedem Zeitpunkt mindestens 3 (verbundene) Partner und sollten 5-10 anstreben. > PeerManagerUserAlert.noConnsTitle=Keine offenen Verbindungen > -PeerManagerUserAlert.noPeersDarknet=Dieser Knoten hat keine Partner mit denen er sich verbinden kann, deshalb wird er nicht normal funktionieren. Im Idealfall sollten Sie sich mit Partner-Knoten verbinden, die von Leuten betrieben werden die Sie kennen (wenn Sie paranoid sind, dann Leute denen Sie vertrauen; wenn nicht, dann zumindest Leute mit denen Sie gesprochen haben). Sie brauchen zu jeder Zeit mindestens 3 verbundene Partner und im Idealfall 5-10. Sie k?nnten sich auf irc.freenode.net im Kanal #freenet-refs einloggen und nach jemandem zum Verbinden fragen, aber bedenken Sie, dass Sie gegen?ber denen, mit denen Sie direkt verbunden sind, verletzbar sind. (Dies gilt vor allem f?r diese fr?he Alpha-Version von Freenet 0.7...) GEHEN SIE SICHER, DASS DIE ANDERE PERSON IHRE REFERENZ AUCH HINZUGEF?GT HAT, DA EINSEITIGE VERBINDUNGEN NICHT FUNKTIONIEREN WERDEN! > -PeerManagerUserAlert.noPeersTestnet=Dieser Knoten hat keine Partner mit denen er sich verbinden kann, deshalb wird er nicht normal funktionieren. Im Idealfall sollten Sie sich mit Partner-Knoten verbinden, die von Leuten betrieben werden die Sie kennen (wenn Sie paranoid sind, dann Leute denen Sie vertrauen; wenn nicht, dann zumindest Leute mit denen Sie gesprochen haben). Sie brauchen zu jeder Zeit mindestens 3 verbundene Partner und im Idealfall 5-10. Aber da dies ein Testnet-Knoten ist, schlagen wir Ihnen vor, sich auf irc.freenode.net im Kanal #freenet-refs einzuloggen und nach jemandem zum Verbinden zu fragen. > PeerManagerUserAlert.noPeersTitle=Keine Partner gefunden > PeerManagerUserAlert.oneConn=Dieser Knoten hat nur eine Verbindung. Die Leistung wird beeintr?chtigt sein und Sie haben weder Anonymit?t noch glaubw?rdige Abstreitbarkeit wenn diese Person b?swillig ist. Ihr Knoten ist wie ein "Blatt" mit dem Netzwerk verbunden und tr?gt nicht zur Gesundheit des Netzwerks bei. Versuchen Sie zu jedem Zeitpunkt mindestens 3 (im Idealfall mehr, wie 5-10) verbundene Partner zu bekommen. > PeerManagerUserAlert.onlyFewConnsTitle=Nur ${count} offene Verbindung(en) > > Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.es.properties > =================================================================== > --- trunk/freenet/src/freenet/l10n/freenet.l10n.es.properties 2008-01-06 13:04:12 UTC (rev 16939) > +++ trunk/freenet/src/freenet/l10n/freenet.l10n.es.properties 2008-01-06 13:08:43 UTC (rev 16940) > @@ -624,8 +624,6 @@ > PeerManagerUserAlert.connErrorTitle=No se puede conectar con algunos contactos > PeerManagerUserAlert.noConns=Este nodo no ha logrado conectar con ning?n otro hasta el momento; no podr? funcionar con normalidad. En caso de que no se establezca ninguna conexi?n pronto, como ser?a deseable, considere obtener nuevos contactos. Se necesitan un m?nimo de 3 en todo momento, siendo ?ptimo tener entre 5 y 10. > PeerManagerUserAlert.noConnsTitle=No hay conexiones establecidas > -PeerManagerUserAlert.noPeersDarknet=Este nodo no tiene contactos a los que conectarse y por tanto no puede funcionar. Idealmente, deber?a conectarse a nodos operados por personas que conozca (y si usted es paranoico, en las que conf?e; si no, al menos personas con las que haya hablado alguna vez). Necesita al menos 3 contactos en todo momento e, idealmente, entre 5 y 10. Puede entrar v?a IRC en irc.freenode.net, canal #freenet-refs y preguntar all? para conseguir alg?n contacto, pero recuerde que es vulnerable ante aquellos a los que se conecte directamente. (Esto es especialmente cierto en esta version preliminar (alfa) de Freenet 0.7...) ASEG?RESE DE QUE LA OTRA PERSONA HA A?ADIDO NUESTRA REFERENCIA, YA QUE LAS CONEXIONES EN UN SOLO SENTIDO NO SIRVEN. > -PeerManagerUserAlert.noPeersTestnet=Este nodo no tiene contactos a los que conectarse, y por tanto no puede funcionar con normalidad. Idealmente, debe conectarse a personas que conozca (si es paranoico, adem?s que sean de su confianza; si no, al menos con las que haya hablado alguna vez). Se necesitan al menos 3 conexiones en todo momento e, idealmente, entre 5 y 10. Pero, dado que este es un nodo de prueba en testnet, se sugiere que conecte con irc.freenode.net, canal #freenet-refs, y pregunte all? por gente que se quiera conectar con usted. > PeerManagerUserAlert.noPeersTitle=No hay contactos > PeerManagerUserAlert.oneConn=Este nodo tiene una ?nica conexi?n. El rendimiento se ver? perjudicado, y usted no cuenta con anonimato o siquiera presunci?n de inocencia si el operador de dicha conexi?n es hostil. Este nodo cuelga de la red como una "hoja" y no contribuye a la salud de la red. Intente obtener un m?nimo de 3 (idealmente entre 5 y 10) conexiones en todo momento. > PeerManagerUserAlert.onlyFewConnsTitle=S?lo hay ${count} conexiones establecidas > > Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties > =================================================================== > --- trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties 2008-01-06 13:04:12 UTC (rev 16939) > +++ trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties 2008-01-06 13:08:43 UTC (rev 16940) > @@ -692,8 +692,6 @@ > PeerManagerUserAlert.connErrorTitle=Alcuni peer non riescono a connettersi > PeerManagerUserAlert.noConns=Non ? stato finora possibile connettersi ad alcun nodo. Forse qualcuno dei peer si connettera entro breve. in caso contrario sar? necessario aggiungere altri peer; c'e' bisogno di almeno tre peer connessi in ogni momento, meglio 5-10. > PeerManagerUserAlert.noConnsTitle=Nessuna connessione aperta > -PeerManagerUserAlert.noPeersDarknet=Questo nodo non ha peers ai quale connettersi e non pu? quindi funzionare normalmente. In teoria ci si dovrebbe connettere esclusivamente a persone che si conosce (per i paranoici: persone di cui ci si fida, per i non paranoici: persone con le quali si ha parlato). Per un corretto funzionamento c'e' bisogno di almeno tre peer connessi in ogni momento, idealmente 5-10. Ci si pu? connettere a irc.freenode.net canale #freenet-refs e chiedere chi vuole connettersi, ma ? bene tenere presente che si ? vulnerabili ad attacchi da parte dei peer (Speciamente in queste prime versioni alfa di Freenet 0.7...) VERIFICARE CHE L'ATRA PARTE AGGIUNGA LA REFERENZA ALLA SUA LISTA: LE CONNESSIONI "A SENSO UNICO" NON FUNZIONANO! > -PeerManagerUserAlert.noPeersTestnet=Questo nodo non ha peer ai quale connettersi e non pu? quindi funzionare normalmente. In teoria ci si dovrebbe connettere esclusivamente a persone che si conosce (per i paranoici: persone di cui ci si fida, per i non paranoici: persone con le quali si ha parlato). Per un corretto funzionamento c'e' bisogno di almeno tre peer connessi in ogni momento, idealmente 5-10. Trattandosi di un nodo testnet, ci si pu? connettere a irc.freenode.net canale #freenet-refs e chiedere chi vuole connettersi. > PeerManagerUserAlert.noPeersTitle=Nessun peer trovato > PeerManagerUserAlert.oneConn=Questo nodo ha una sola connessione. Il rendimento ne risentir? in modo notevole, e l'utente non disporr? di anonimato e "negabilit? plausibile" se quell' unico nodo al quale si ? conessi dovesse essere operato da un avversario. Il nodo risulter? attaccato al network come una "foglia all' albero" e non contribuir? alla salute generale del network stesso. Per un corretto funzionamento del nodo ? necessario che almeno tre peer e (idealmente 5-10) siano connessi in qualsiasi momento. > PeerManagerUserAlert.onlyFewConnsTitle=Soltanto ${count} connessione/i aperta/e > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/d8fabc36/attachment.pgp From freenet-devl at david.sowder.com Fri Jan 11 19:49:09 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Fri, 11 Jan 2008 13:49:09 -0600 Subject: [freenet-dev] Message priorities (and coalescing timeouts) was Re: [freenet-cvs] r16960 - trunk/freenet/src/freenet/node In-Reply-To: <200801111740.35908.toad@amphibian.dyndns.org> References: <20080107204520.4EAB139230A@freenetproject.org> <20080111000643.3ptzwun1mskc84gc@69.49.230.6> <47872A09.40109@cs.ucl.ac.uk> <200801111740.35908.toad@amphibian.dyndns.org> Message-ID: <4787C835.4060102@david.sowder.com> Matthew Toseland wrote: > On Friday 11 January 2008 08:34, Michael Rogers wrote: > >> Robert Hailey wrote: >> >>> Rejecting new >>> requests would keep the data-transfer level traffic from being >>> starved. >>> >> Yup, that makes sense as long as it also applies to local requests. >> > > Which it does, as of some weeks back. > > BTW robert, we do have a limit on the number of running requests: the > bandwidth liability limits. > > Now, is there any reason to prioritise different control messages differently? > And do we want absolute priorities, or different timeouts (messages will be > sent at t+100ms if they haven't been sent already at the moment, we could > vary this by type), or both? > > For example: > > Group 0: > 100ms coalescing timeout. > Packet acknowledgments. > These need to be sent fairly quickly. > > > Group 2: > Messages which don't have particularly short timeouts. > 200ms coalescing timeout? Or even more? > > nodeToNodeMessage - maybe treat this as bulk I would argue that nodeToNodeMessage should not be at any lower priority because it is now not only used for potentially pseudo-realtime chat but also "control" messages in the form of differential node references. From robert at emu.freenetproject.org Fri Jan 11 19:53:50 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 13:53:50 -0600 Subject: [freenet-dev] r16886 - trunk/freenet/src/freenet/node In-Reply-To: <200801111843.03060.toad@amphibian.dyndns.org> References: <20080104182211.0332D390A59@freenetproject.org> <200801082027.32705.toad@amphibian.dyndns.org> <0CC1E253-3C9F-483A-8137-D43B74F2E86B@emu.freenetproject.org> <200801111843.03060.toad@amphibian.dyndns.org> Message-ID: On Jan 11, 2008, at 12:22 PM, Matthew Toseland wrote: > On Tuesday 08 January 2008 22:57, Robert Hailey wrote: >> Jan 08, 2008 20:03:41:146 (freenet.node.RequestSender, RequestSender >> for UID -3998139406700477577, ERROR): discontinuing non-local request >> search, general timeout (6 attempts, 3 overloads) >> ... >> Jan 08, 2008 20:12:21:226 (freenet.node.RequestSender, RequestSender >> for UID 60170596711015291, ERROR): discontinuing non-local request >> search, general timeout (1 attempts, 3 overloads) > > Ouch. How common is this? The super-majority of requests which took to long asked only one peer (~65%), but this does not speak to how many completed correctly. >> Presently the node will "RejectLoop" if it is one of the last 10000 >> completed requests. My node runs through that many requests in about >> 16 minutes. With this logging statement it is already shown that a >> request can last longer than 2 minutes for one peer (and most nodes >> have 20). If you assume that a request takes 4 minutes (two peers, >> VERY optimistic), then it would then only take 4 nodes ('near' each >> other) to generate a request-live-lock (absent the HTL; the request >> would never drop from the network); each trying two of it's other >> peers, and then the next in the 4-chain. > > Okay, this is a problem. On Jan 11, 2008, at 12:42 PM, Matthew Toseland wrote: > On Wednesday 09 January 2008 17:14, Robert Hailey wrote: >> >> I have reverted r16886, as it appears to be based on a misunderstand >> of how requests work against the topology of the network (r16980). > > I was rather hoping to be talked into accepting 16886 ... we should > at least > have some logging in such cases IMHO. > > Requests really shouldn't be taking that long - maybe it's related > to the HTL > problem, maybe we have such a perverse network topology that we are > resetting > HTL time after time after time, I dunno, simulations would be > interesting. I'm thinking that message queue priorities will obsolete this problem, as the path for responses to requests will solidify nearly immediately. Which is to say, we will then see mostly fetch-timeouts, not accepted/fatal-timeouts. >> Instead of canceling the search request, I have put similar logic in >> RequestHandler to simply not respond to very old requests (r16982), > > Ok. Maybe this is where we should have such logging. Agreed, r17017. I've put it in at 'error' level, so it will be a bit chatty until priorities come into play. >> and not to hang onto the thread if no response is therefore necessary >> (r16983). >> >> r16983, (given the timeout) makes extra calls to >> rejectedOverload(); I >> imagine that at the timeout (every 2 minutes while the local request >> is running) they are benign, but I can code up an infinite wait (as >> before) if needed. > > Well it's not strictly infinite - it's 2min*peers at most. You > reverted this > in 16985. >> >> I'm going to try and stay away from high level changes for a while, >> but at your suggestion I will look at the HTL code. > > You are more than welcome to continue to investigate the timeouts, > they have > been a problem for a long time. Do they still show up in simulation > with the > HTL problem fixed? I'm very happy to have somebody else working on > core > stuff, I'm equally happy to point out any obvious problems with their > commits. :) Why thanks! -- Robert Hailey From david at sowder.com Fri Jan 11 19:55:13 2008 From: david at sowder.com (David Sowder) Date: Fri, 11 Jan 2008 13:55:13 -0600 Subject: [freenet-dev] [freenet-cvs] r16954 - trunk/freenet/src/freenet/node In-Reply-To: <200801102256.56037.toad@amphibian.dyndns.org> References: <20080106225149.A94CB3C0CE8@freenetproject.org> <200801102256.56037.toad@amphibian.dyndns.org> Message-ID: <4787C9A1.2090303@sowder.com> We probably should define a range. I'd say 0-9999 are reserved for the node and 65000+ are reserved for experimental, "unregistered" use. 10000-64999 are available to be "well known" types registered through a request to a dev with commit access to the node's repo. As for FCP access to N2NMs, I've planned that for a long while now, but my plans are a little elaborate and I haven't really started sorting out the details in my head or anything. Perhaps the bug will bite in the coming weeks. Matthew Toseland wrote: > We should really define a range which can be used by the node, so that numbers > above that can be used by applications when we implement FCP layer use of > this mechanism (which IMHO is long overdue and would open up some interesting > darknet collaboration possibilities e.g. send bookmark to peer, send thaw > index to peer etc). > > On Sunday 06 January 2008 22:51, zothar at freenetproject.org wrote: > >> Author: zothar >> Date: 2008-01-06 22:51:49 +0000 (Sun, 06 Jan 2008) >> New Revision: 16954 >> >> Modified: >> trunk/freenet/src/freenet/node/Node.java >> trunk/freenet/src/freenet/node/PeerNode.java >> Log: >> Implement processing of received differential node references. It may be a >> > bit before I get sending implemented. > >> Modified: trunk/freenet/src/freenet/node/Node.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/Node.java 2008-01-06 22:09:51 UTC (rev >> > 16953) > >> +++ trunk/freenet/src/freenet/node/Node.java 2008-01-06 22:51:49 UTC (rev >> > 16954) > >> @@ -2576,8 +2576,20 @@ >> throw new Error(e); >> } >> } else if(type == Node.N2N_MESSAGE_TYPE_DIFFNODEREF) { >> - // FIXME: Not yet implemented >> Logger.normal(this, "Received differential node reference node to node >> > message from "+src.getPeer()); > >> + SimpleFieldSet fs = null; >> + try { >> + fs = new SimpleFieldSet(new String(messageData.getData(), "UTF-8"), >> > false, true); > >> + } catch (IOException e) { >> + Logger.error(this, "IOException while parsing node to node message >> > data", e); > >> + return; >> + } >> + try { >> + src.processDiffNoderef(fs); >> + } catch (FSParseException e) { >> + Logger.error(this, "FSParseException while parsing node to node message >> > data", e); > >> + return; >> + } >> } else { >> Logger.error(this, "Received unknown node to node message >> > type '"+type+"' from "+src.getPeer()); > >> } >> >> Modified: trunk/freenet/src/freenet/node/PeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-06 22:09:51 UTC >> > (rev 16953) > >> +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-06 22:51:49 UTC >> > (rev 16954) > >> @@ -1934,6 +1934,14 @@ >> } >> >> /** >> + * Process a differential node reference >> + * The identity must not change, or we throw. >> + */ >> + public void processDiffNoderef(SimpleFieldSet fs) throws FSParseException >> > { > >> + processNewNoderef(fs, false, true); >> + } >> + >> + /** >> * Process a new nodereference, in compressed form. >> * The identity must not change, or we throw. >> */ >> >> _______________________________________________ >> cvs mailing list >> cvs at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs >> >> >> >> ------------------------------------------------------------------------ >> >> _______________________________________________ >> Devl mailing list >> Devl at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl -- David R. Sowder, Linux Systems Admin (Software Systems Specialist II) Office of Information Technology, University of Texas at Arlington Work: 817-272-1081 davids at uta.edu http://www.uta.edu/ davids at talk.uta.edu (Jabber) Personal: david at sowder.com http://david.sowder.com/ From freenet-devl at david.sowder.com Fri Jan 11 20:05:41 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Fri, 11 Jan 2008 14:05:41 -0600 Subject: [freenet-dev] [freenet-cvs] r16974 - trunk/freenet/src/freenet/node In-Reply-To: <200801110015.16489.toad@amphibian.dyndns.org> References: <20080109005307.94EB747A0E3@freenetproject.org> <200801110015.16489.toad@amphibian.dyndns.org> Message-ID: <4787CC15.6070204@david.sowder.com> That is exactly the reason for both: seed nodes are technically neither. Matthew Toseland wrote: > Is there any reason to have both isDarknet() and isOpennet()? I suppose seed > nodes would return false for both? > > On Wednesday 09 January 2008 00:53, zothar at freenetproject.org wrote: > >> Author: zothar >> Date: 2008-01-09 00:53:07 +0000 (Wed, 09 Jan 2008) >> New Revision: 16974 >> >> Modified: >> trunk/freenet/src/freenet/node/DarknetPeerNode.java >> trunk/freenet/src/freenet/node/OpennetPeerNode.java >> trunk/freenet/src/freenet/node/PeerNode.java >> trunk/freenet/src/freenet/node/SeedClientPeerNode.java >> trunk/freenet/src/freenet/node/SeedServerPeerNode.java >> Log: >> Implement isDarknet(). Move sendNodeToNodeMessage() to PeerNode and queue >> > on not connected only if the peer is a darknet peer. > >> Modified: trunk/freenet/src/freenet/node/DarknetPeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/DarknetPeerNode.java 2008-01-09 00:00:39 >> > UTC (rev 16973) > >> +++ trunk/freenet/src/freenet/node/DarknetPeerNode.java 2008-01-09 00:53:07 >> > UTC (rev 16974) > >> @@ -1257,28 +1257,6 @@ >> } >> } >> >> - public void sendNodeToNodeMessage(SimpleFieldSet fs, int n2nType, boolean >> > includeSentTime, long now) { > >> - fs.put("n2nType", n2nType); >> - if(includeSentTime) { >> - fs.put("sentTime", now); >> - } >> - try { >> - Message n2nm; >> - n2nm = DMT.createNodeToNodeMessage( >> - n2nType, fs.toString().getBytes("UTF-8")); >> - try { >> - sendAsync(n2nm, null, 0, null); >> - } catch (NotConnectedException e) { >> - if(includeSentTime) { >> - fs.removeValue("sentTime"); >> - } >> - queueN2NM(fs); >> - } >> - } catch (UnsupportedEncodingException e) { >> - throw new Error("Impossible: "+e, e); >> - } >> - } >> - >> public int sendFileOfferAccepted(long uid) { >> storeOffers(); >> long now = System.currentTimeMillis(); >> @@ -1509,6 +1487,10 @@ >> return new DarknetPeerNodeStatus(this, noHeavy); >> } >> >> + public boolean isDarknet() { >> + return true; >> + } >> + >> public boolean isOpennet() { >> return false; >> } >> >> Modified: trunk/freenet/src/freenet/node/OpennetPeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/OpennetPeerNode.java 2008-01-09 00:00:39 >> > UTC (rev 16973) > >> +++ trunk/freenet/src/freenet/node/OpennetPeerNode.java 2008-01-09 00:53:07 >> > UTC (rev 16974) > >> @@ -23,6 +23,10 @@ >> return super.isRoutingCompatible(); >> } >> >> + public boolean isDarknet() { >> + return false; >> + } >> + >> public boolean isOpennet() { >> return true; >> } >> >> Modified: trunk/freenet/src/freenet/node/PeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-09 00:00:39 UTC >> > (rev 16973) > >> +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-09 00:53:07 UTC >> > (rev 16974) > >> @@ -2367,6 +2367,8 @@ >> return fs; >> } >> >> + public abstract boolean isDarknet(); >> + >> public abstract boolean isOpennet(); >> >> /** >> @@ -3498,4 +3500,35 @@ >> } >> >> private int handshakeIPAlternator; >> + >> + public void sendNodeToNodeMessage(SimpleFieldSet fs, int n2nType, boolean >> > includeSentTime, long now) { > >> + fs.put("n2nType", n2nType); >> + if(includeSentTime) { >> + fs.put("sentTime", now); >> + } >> + try { >> + Message n2nm; >> + n2nm = DMT.createNodeToNodeMessage( >> + n2nType, fs.toString().getBytes("UTF-8")); >> + try { >> + sendAsync(n2nm, null, 0, null); >> + } catch (NotConnectedException e) { >> + if(includeSentTime) { >> + fs.removeValue("sentTime"); >> + } >> + if(isDarknet()) { >> + queueN2NM(fs); >> + } >> + } >> + } catch (UnsupportedEncodingException e) { >> + throw new Error("Impossible: "+e, e); >> + } >> + } >> + >> + /** >> + * A method to be to queue an N2NM in a extra peer data file, only >> > implemented by DarknetPeerNode > >> + */ >> + public void queueN2NM(SimpleFieldSet fs) { >> + // Do nothing in the default impl >> + } >> } >> >> Modified: trunk/freenet/src/freenet/node/SeedClientPeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/SeedClientPeerNode.java 2008-01-09 >> > 00:00:39 UTC (rev 16973) > >> +++ trunk/freenet/src/freenet/node/SeedClientPeerNode.java 2008-01-09 >> > 00:53:07 UTC (rev 16974) > >> @@ -21,6 +21,10 @@ >> return new PeerNodeStatus(this, noHeavy); >> } >> >> + public boolean isDarknet() { >> + return false; >> + } >> + >> public boolean isOpennet() { >> return false; // Not exactly >> } >> >> Modified: trunk/freenet/src/freenet/node/SeedServerPeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/SeedServerPeerNode.java 2008-01-09 >> > 00:00:39 UTC (rev 16973) > >> +++ trunk/freenet/src/freenet/node/SeedServerPeerNode.java 2008-01-09 >> > 00:53:07 UTC (rev 16974) > >> @@ -26,6 +26,10 @@ >> return new PeerNodeStatus(this, noHeavy); >> } >> >> + public boolean isDarknet() { >> + return false; >> + } >> + >> public boolean isOpennet() { >> return false; >> } >> >> _______________________________________________ >> cvs mailing list >> cvs at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs >> >> >> >> ------------------------------------------------------------------------ >> >> _______________________________________________ >> Devl mailing list >> Devl at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From nextgens at freenetproject.org Fri Jan 11 20:11:12 2008 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Fri, 11 Jan 2008 21:11:12 +0100 Subject: [freenet-dev] [freenet-cvs] r16940 - trunk/freenet/src/freenet/l10n In-Reply-To: <200801111937.19777.toad@amphibian.dyndns.org> References: <20080106130844.6351147B849@freenetproject.org> <200801111937.19777.toad@amphibian.dyndns.org> Message-ID: <20080111201112.GG4303@freenetproject.org> * Matthew Toseland [2008-01-11 19:37:19]: > On Sunday 06 January 2008 13:08, you wrote: > > Author: nextgens > > Date: 2008-01-06 13:08:43 +0000 (Sun, 06 Jan 2008) > > New Revision: 16940 > > > > Modified: > > trunk/freenet/src/freenet/l10n/freenet.l10n.de.properties > > trunk/freenet/src/freenet/l10n/freenet.l10n.es.properties > > trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties > > Log: > > l10n: sed -i '/#freenet/d' *properties ... > > > > it\'s better to have an incomplete translation than an outdated one > > Are these going to be updated? Do we have maintainers for these languages? They should be... deleting them was probably wiser than keeping them anyway. NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/8a7d18fa/attachment.pgp From freenet-devl at david.sowder.com Fri Jan 11 20:15:25 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Fri, 11 Jan 2008 14:15:25 -0600 Subject: [freenet-dev] [freenet-cvs] r16974 - trunk/freenet/src/freenet/node In-Reply-To: <200801102354.10975.toad@amphibian.dyndns.org> References: <20080109005307.94EB747A0E3@freenetproject.org> <200801102354.10975.toad@amphibian.dyndns.org> Message-ID: <4787CE5D.90305@david.sowder.com> Matthew Toseland wrote: > On Wednesday 09 January 2008 00:53, zothar at freenetproject.org wrote: > >> Author: zothar >> Date: 2008-01-09 00:53:07 +0000 (Wed, 09 Jan 2008) >> New Revision: 16974 >> >> Modified: >> trunk/freenet/src/freenet/node/DarknetPeerNode.java >> trunk/freenet/src/freenet/node/OpennetPeerNode.java >> trunk/freenet/src/freenet/node/PeerNode.java >> trunk/freenet/src/freenet/node/SeedClientPeerNode.java >> trunk/freenet/src/freenet/node/SeedServerPeerNode.java >> Log: >> Implement isDarknet(). Move sendNodeToNodeMessage() to PeerNode and queue >> > on not connected only if the peer is a darknet peer. > > This is getting awfully complicated. > It doesn't seem particularly complicated to me, but then I just implemented it. > I had assumed that N2NMs were a mechanism for clients (including fproxy, which > is an internal client) to communicate with each other. N2NMs are therefore > only useful on darknet, since on opennet there is no reason to talk to other > local clients - certainly this reasoning applies to fproxy text messages, and > certainly it applies to queueing messages. On the other hand, there may be > some applications for which being able to talk to your opposite client on > your opennet peers may be useful - namely any application that implements its > own routed protocol on top of Freenet. Many of these (e.g. currency over > freenet) require a trust connection - but maybe some wouldn't (e.g. > broadcasting). I had assumed that not providing any N2NM service for opennet > peers was a sensible default, which we could change when a client author asks > for it. But perhaps I was wrong? One obvious limitation with allowing N2NMs > to/from opennet peers is that queueing is probably a bad idea. However, once > we have FCP clients, we might want to provide a way to send an N2NM and not > queue it if the node is disconnected, just drop it, because it's not > important if it's not delivered quickly. This can easily be implemented on > top of queueing. > I'm using N2NMs as a generic message type and I thought you were cool with that after our earlier discussion about differential node references. Because of this, I don't assume all functionality of N2NMs, as only previously used by N2NTMs, to be darknet-only. As for own routing on top of Freenet, that's one of the things I've been wanting to play with once I got N2NMs into FCP. My current assumption, and my implementation reflects it, is that queuing a N2NM for an opennet peer doesn't make sense because we have no idea if we'll ever re-connect to that currently disconnected peer. > So the simplest solution is to always queue, to only allow N2NMs to/from > darknet peers, and to have a separate message type for the low-level > mechanism of differential noderefs (which can of course share code if there > is any major code to be done). Is the simplest solution the best in this > case? It seems like premature complexity to me - but maybe it's useful > complexity? > I don't always queue because queuing is only needed for disconnected peers, though I supposed we could sendAsync it and then the connection dies before it's delivered. Currently, I like the idea of N2NMs not being darknet-only. Sure we could say they are now, change differential node refs to a dedicated message and give them opennet access when someone asks for it, but implementing that might be harder then if it's got the darknet-only assumption even more coded in than it was. To me, this commit is quite straight-forward, but then again, maybe I don't see the complexity because I coded it. >> Modified: trunk/freenet/src/freenet/node/DarknetPeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/DarknetPeerNode.java 2008-01-09 00:00:39 >> > UTC (rev 16973) > >> +++ trunk/freenet/src/freenet/node/DarknetPeerNode.java 2008-01-09 00:53:07 >> > UTC (rev 16974) > >> @@ -1257,28 +1257,6 @@ >> } >> } >> >> - public void sendNodeToNodeMessage(SimpleFieldSet fs, int n2nType, boolean >> > includeSentTime, long now) { > >> - fs.put("n2nType", n2nType); >> - if(includeSentTime) { >> - fs.put("sentTime", now); >> - } >> - try { >> - Message n2nm; >> - n2nm = DMT.createNodeToNodeMessage( >> - n2nType, fs.toString().getBytes("UTF-8")); >> - try { >> - sendAsync(n2nm, null, 0, null); >> - } catch (NotConnectedException e) { >> - if(includeSentTime) { >> - fs.removeValue("sentTime"); >> - } >> - queueN2NM(fs); >> - } >> - } catch (UnsupportedEncodingException e) { >> - throw new Error("Impossible: "+e, e); >> - } >> - } >> - >> public int sendFileOfferAccepted(long uid) { >> storeOffers(); >> long now = System.currentTimeMillis(); >> @@ -1509,6 +1487,10 @@ >> return new DarknetPeerNodeStatus(this, noHeavy); >> } >> >> + public boolean isDarknet() { >> + return true; >> + } >> + >> public boolean isOpennet() { >> return false; >> } >> >> Modified: trunk/freenet/src/freenet/node/OpennetPeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/OpennetPeerNode.java 2008-01-09 00:00:39 >> > UTC (rev 16973) > >> +++ trunk/freenet/src/freenet/node/OpennetPeerNode.java 2008-01-09 00:53:07 >> > UTC (rev 16974) > >> @@ -23,6 +23,10 @@ >> return super.isRoutingCompatible(); >> } >> >> + public boolean isDarknet() { >> + return false; >> + } >> + >> public boolean isOpennet() { >> return true; >> } >> >> Modified: trunk/freenet/src/freenet/node/PeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-09 00:00:39 UTC >> > (rev 16973) > >> +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-09 00:53:07 UTC >> > (rev 16974) > >> @@ -2367,6 +2367,8 @@ >> return fs; >> } >> >> + public abstract boolean isDarknet(); >> + >> public abstract boolean isOpennet(); >> >> /** >> @@ -3498,4 +3500,35 @@ >> } >> >> private int handshakeIPAlternator; >> + >> + public void sendNodeToNodeMessage(SimpleFieldSet fs, int n2nType, boolean >> > includeSentTime, long now) { > >> + fs.put("n2nType", n2nType); >> + if(includeSentTime) { >> + fs.put("sentTime", now); >> + } >> + try { >> + Message n2nm; >> + n2nm = DMT.createNodeToNodeMessage( >> + n2nType, fs.toString().getBytes("UTF-8")); >> + try { >> + sendAsync(n2nm, null, 0, null); >> + } catch (NotConnectedException e) { >> + if(includeSentTime) { >> + fs.removeValue("sentTime"); >> + } >> + if(isDarknet()) { >> + queueN2NM(fs); >> + } >> + } >> + } catch (UnsupportedEncodingException e) { >> + throw new Error("Impossible: "+e, e); >> + } >> + } >> + >> + /** >> + * A method to be to queue an N2NM in a extra peer data file, only >> > implemented by DarknetPeerNode > >> + */ >> + public void queueN2NM(SimpleFieldSet fs) { >> + // Do nothing in the default impl >> + } >> } >> >> Modified: trunk/freenet/src/freenet/node/SeedClientPeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/SeedClientPeerNode.java 2008-01-09 >> > 00:00:39 UTC (rev 16973) > >> +++ trunk/freenet/src/freenet/node/SeedClientPeerNode.java 2008-01-09 >> > 00:53:07 UTC (rev 16974) > >> @@ -21,6 +21,10 @@ >> return new PeerNodeStatus(this, noHeavy); >> } >> >> + public boolean isDarknet() { >> + return false; >> + } >> + >> public boolean isOpennet() { >> return false; // Not exactly >> } >> >> Modified: trunk/freenet/src/freenet/node/SeedServerPeerNode.java >> =================================================================== >> --- trunk/freenet/src/freenet/node/SeedServerPeerNode.java 2008-01-09 >> > 00:00:39 UTC (rev 16973) > >> +++ trunk/freenet/src/freenet/node/SeedServerPeerNode.java 2008-01-09 >> > 00:53:07 UTC (rev 16974) > >> @@ -26,6 +26,10 @@ >> return new PeerNodeStatus(this, noHeavy); >> } >> >> + public boolean isDarknet() { >> + return false; >> + } >> + >> public boolean isOpennet() { >> return false; >> } >> >> _______________________________________________ >> cvs mailing list >> cvs at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs >> >> >> >> ------------------------------------------------------------------------ >> >> _______________________________________________ >> Devl mailing list >> Devl at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From freenet-devl at david.sowder.com Fri Jan 11 20:19:30 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Fri, 11 Jan 2008 14:19:30 -0600 Subject: [freenet-dev] [freenet-cvs] r16974 - trunk/freenet/src/freenet/node In-Reply-To: <4786B83F.6010406@cs.ucl.ac.uk> References: <20080109005307.94EB747A0E3@freenetproject.org> <200801102354.10975.toad@amphibian.dyndns.org> <4786B83F.6010406@cs.ucl.ac.uk> Message-ID: <4787CF52.4000707@david.sowder.com> Yeah, someone had already suggested N2NTMs between opennet peers on IRC, but to me, that would not only discourage the adoption of darknet peerings but also, for purposes of establishing a darknet connection, it would be starting at the extreme opposite end of the trust scale (in my book) than a darknet peering needs to be, rather than an otherwise neutral position. Imagine attackers using opennet to pick your node and then using N2NTMs to build a "trust" relationship with you to get a darknet peering with you, which you might then think of a "trustworthy" and "secure". (I'd definitely have the fact that I met them on opennet in my private peer note on that peer.) Michael Rogers wrote: > Matthew Toseland wrote: > >> I had assumed that N2NMs were a mechanism for clients (including fproxy, which >> is an internal client) to communicate with each other. N2NMs are therefore >> only useful on darknet, since on opennet there is no reason to talk to other >> local clients - >> > > I was going to suggest N2N chat between opennet peers, perhaps with an > eye to establishing a darknet connection. But there's a danger that, > since it's Freenet, users will assume they're anonymous when chatting... > and there doesn't seem much point offering a friendly service like N2N > chat if it has to be plastered in warnings. > From m.rogers at cs.ucl.ac.uk Fri Jan 11 20:20:57 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 11 Jan 2008 20:20:57 +0000 Subject: [freenet-dev] [freenet-cvs] r16954 - trunk/freenet/src/freenet/node In-Reply-To: <4787C9A1.2090303@sowder.com> References: <20080106225149.A94CB3C0CE8@freenetproject.org> <200801102256.56037.toad@amphibian.dyndns.org> <4787C9A1.2090303@sowder.com> Message-ID: <4787CFA9.1090406@cs.ucl.ac.uk> David Sowder wrote: > We probably should define a range. I'd say 0-9999 are reserved for the > node and 65000+ are reserved for experimental, "unregistered" use. > 10000-64999 are available to be "well known" types registered through a > request to a dev with commit access to the node's repo. Instead of well-known numbers, how about allowing clients to register service names (on a first-come-first-served basis, automatically deregistered when the client disconnects), with a simple name->number lookup service for the remote client? In other words why not imitate Bonjour rather than IANA? Cheers, Michael From freenet-devl at david.sowder.com Fri Jan 11 20:20:24 2008 From: freenet-devl at david.sowder.com (David Sowder) Date: Fri, 11 Jan 2008 14:20:24 -0600 Subject: [freenet-dev] [freenet-cvs] r17015 - trunk/freenet/src/freenet/io/comm In-Reply-To: <200801111741.32653.toad@amphibian.dyndns.org> References: <20080111170823.585423A19DA@freenetproject.org> <200801111741.32653.toad@amphibian.dyndns.org> Message-ID: <4787CF88.3050100@david.sowder.com> Matthew Toseland wrote: > On Friday 11 January 2008 17:18, Robert Hailey wrote: > >> On Jan 11, 2008, at 11:08 AM, toad at freenetproject.org wrote: >> >> >>> Author: toad >>> Date: 2008-01-11 17:08:23 +0000 (Fri, 11 Jan 2008) >>> New Revision: 17015 >>> >>> Modified: >>> trunk/freenet/src/freenet/io/comm/MessageCore.java >>> Log: >>> delete ping handling >>> >> Won't this break the simulator again? >> > > Will it? I didn't see any references, but if you need it reinstate it. > I seem to remember it being dead code back when I was looking into the possible reasons for unclaimedFIFOSize growing. From m.rogers at cs.ucl.ac.uk Fri Jan 11 20:52:39 2008 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Fri, 11 Jan 2008 20:52:39 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <4787BE1B.8090508@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <200801041410.22489.toad@amphibian.dyndns.org> <477E7E1D.9060300@cs.ucl.ac.uk> <200801111845.27956.toad@amphibian.dyndns.org> <4787BE1B.8090508@cs.ucl.ac.uk> Message-ID: <4787D717.1020808@cs.ucl.ac.uk> Michael Rogers wrote: > I have some notes somewhere on various ways of defining cluster and > cliques... I'll see if there's anything that could be used to agree on > the membership of a cell. Two approaches from Knoke & Kuklinski: 1) n-cliques: every member can reach every other in n hops. (We'd need to decide whether paths may pass through non-members for the purposes of calculating the membership.) 2) k-plex cliques: each of the n members is connected to at least n-k others (so a fully connected subgraph is a 1-plex clique). But in either case I'm not sure how to work out which cliques a node belongs to using only its local view of the network. In the k-plex case, what if the clique is larger than my local view? In the n-clique case, if two nodes within n hops of me aren't within n hops of each other, which do I eliminate? Cheers, Michael From robert at emu.freenetproject.org Fri Jan 11 21:22:24 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 15:22:24 -0600 Subject: [freenet-dev] Message priorities (and coalescing timeouts) In-Reply-To: <200801111818.59091.toad@amphibian.dyndns.org> References: <20080107204520.4EAB139230A@freenetproject.org> <200801111740.35908.toad@amphibian.dyndns.org> <66392E4E-7279-49C4-B16B-FB7BCFCE6EA8@emu.freenetproject.org> <200801111818.59091.toad@amphibian.dyndns.org> Message-ID: <97CF0B6D-2F1A-414F-928A-A52B6BBEABCF@emu.freenetproject.org> On Jan 11, 2008, at 12:18 PM, Matthew Toseland wrote: > Are you interested in implementing message priorities? MessageItem and > PacketSender are the most relevant classes. I'll certainly do the grunt work of putting in the priority constants and accessors in (r17019), but I kinda don't want to touch FNPPacketMangler/PacketSender. Also, the packet acknowledgements appear to be a lower level than the message queue. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080111/af177c46/attachment.htm From toad at amphibian.dyndns.org Fri Jan 11 21:55:33 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 21:55:33 +0000 Subject: [freenet-dev] =?iso-8859-1?q?=5Bfreenet-cvs=5D_r16974_-=09trunk/f?= =?iso-8859-1?q?reenet/src/freenet/node?= In-Reply-To: <4787CE5D.90305@david.sowder.com> References: <20080109005307.94EB747A0E3@freenetproject.org> <200801102354.10975.toad@amphibian.dyndns.org> <4787CE5D.90305@david.sowder.com> Message-ID: <200801112155.40044.toad@amphibian.dyndns.org> On Friday 11 January 2008 20:15, David Sowder wrote: > I'm using N2NMs as a generic message type and I thought you were cool > with that after our earlier discussion about differential node > references. Because of this, I don't assume all functionality of N2NMs, > as only previously used by N2NTMs, to be darknet-only. As for own > routing on top of Freenet, that's one of the things I've been wanting to > play with once I got N2NMs into FCP. My current assumption, and my > implementation reflects it, is that queuing a N2NM for an opennet peer > doesn't make sense because we have no idea if we'll ever re-connect to > that currently disconnected peer. > > So the simplest solution is to always queue, to only allow N2NMs to/from > > darknet peers, and to have a separate message type for the low-level > > mechanism of differential noderefs (which can of course share code if there > > is any major code to be done). Is the simplest solution the best in this > > case? It seems like premature complexity to me - but maybe it's useful > > complexity? > > > I don't always queue because queuing is only needed for disconnected > peers, though I supposed we could sendAsync it and then the connection > dies before it's delivered. > Currently, I like the idea of N2NMs not being darknet-only. Sure we > could say they are now, change differential node refs to a dedicated > message and give them opennet access when someone asks for it, but > implementing that might be harder then if it's got the darknet-only > assumption even more coded in than it was. > > To me, this commit is quite straight-forward, but then again, maybe I > don't see the complexity because I coded it. The question is does it make the primary use of N2NMs - use by clients over FCP - more difficult? -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/1e07d863/attachment.pgp From robert at emu.freenetproject.org Fri Jan 11 22:23:44 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 16:23:44 -0600 Subject: [freenet-dev] Message priorities (and coalescing timeouts) was Re: [freenet-cvs] r16960 - trunk/freenet/src/freenet/node In-Reply-To: <200801111740.35908.toad@amphibian.dyndns.org> References: <20080107204520.4EAB139230A@freenetproject.org> <20080111000643.3ptzwun1mskc84gc@69.49.230.6> <47872A09.40109@cs.ucl.ac.uk> <200801111740.35908.toad@amphibian.dyndns.org> Message-ID: <9D4E3198-1A5A-4178-B371-0F489C9847D9@emu.freenetproject.org> On Jan 11, 2008, at 11:40 AM, Matthew Toseland wrote: > Group 1: > 100ms coalescing timeout. > Various messages with short timeouts, or which we need to send > quickly for one > reason or another. > ... > FNPPing/FNPPong - ???? > FNPLinkPing/FNPLinkPong - ???? Good or bad, I don't know... but in order for [SUB_]MAX_PING_TIME to be the same stat the pings would either have to be queued with the bulk data or a separate data-level-ping be made. Presently I suppose it gives the combined send queue lengths (to/from) + rtt (x2), and the caps are 700ms & 1500ms. -- Robert Hailey -------------- next part -------------- An HTML attachment was scrubbed... URL: http://emu.freenetproject.org/pipermail/devl/attachments/20080111/3070b2fd/attachment.htm From toad at amphibian.dyndns.org Fri Jan 11 22:06:24 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 11 Jan 2008 22:06:24 +0000 Subject: [freenet-dev] [freenet-cvs] r16987 - trunk/freenet/src/freenet/io/xfer In-Reply-To: <20080109221037.8B642479874@freenetproject.org> References: <20080109221037.8B642479874@freenetproject.org> Message-ID: <200801112206.24857.toad@amphibian.dyndns.org> On Wednesday 09 January 2008 22:10, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-09 22:10:36 +0000 (Wed, 09 Jan 2008) > New Revision: 16987 > > Modified: > trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > Log: > don't assume everything has been transmitted when the transmitter oversleeps > - should get rid of "haven't heard from receiver in 1m55s." message > - the transmitter still oversleeps (2 minutes?!), and is not woken up by notifyAll() because it is in the throttle function: delay() Why does the below help? > > > Modified: trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > =================================================================== > --- trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java 2008-01-09 18:33:46 UTC (rev 16986) > +++ trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java 2008-01-09 22:10:36 UTC (rev 16987) > @@ -192,6 +192,7 @@ > public void packetReceived(int packetNo) { > synchronized(_senderThread) { > _unsent.addLast(new Integer(packetNo)); > + timeAllSent = -1; > _sentPackets.setBit(packetNo, false); > _senderThread.notifyAll(); > } -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080111/f2185e03/attachment.pgp From robert at emu.freenetproject.org Fri Jan 11 22:46:24 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 16:46:24 -0600 Subject: [freenet-dev] [freenet-cvs] r16987 - trunk/freenet/src/freenet/io/xfer In-Reply-To: <200801112206.24857.toad@amphibian.dyndns.org> References: <20080109221037.8B642479874@freenetproject.org> <200801112206.24857.toad@amphibian.dyndns.org> Message-ID: <79F840DD-6D9C-4515-9214-B8B9F6318FBE@emu.freenetproject.org> On Jan 11, 2008, at 4:06 PM, Matthew Toseland wrote: > On Wednesday 09 January 2008 22:10, robert at freenetproject.org wrote: >> Author: robert >> Date: 2008-01-09 22:10:36 +0000 (Wed, 09 Jan 2008) >> New Revision: 16987 >> >> Modified: >> trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java >> Log: >> don't assume everything has been transmitted when the transmitter >> oversleeps >> - should get rid of "haven't heard from receiver in 1m55s." message >> - the transmitter still oversleeps (2 minutes?!), and is not woken >> up by > notifyAll() because it is in the throttle function: delay() > > Why does the below help? I think it was actually because delay() was deadlocking, not oversleeping. In any case, this helps then because we will not then see an old timeAllSent and assume that the receiver is dead ("haven't heard from the receiver"). -- Robert Hailey > >> >> >> Modified: trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java >> =================================================================== >> --- trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java >> 2008-01-09 > 18:33:46 UTC (rev 16986) >> +++ trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java >> 2008-01-09 > 22:10:36 UTC (rev 16987) >> @@ -192,6 +192,7 @@ >> public void packetReceived(int packetNo) { >> synchronized(_senderThread) { >> _unsent.addLast(new Integer(packetNo)); >> + timeAllSent = -1; >> _sentPackets.setBit(packetNo, false); >> _senderThread.notifyAll(); >> } > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From tommy100 at gmx.de Fri Jan 11 23:31:56 2008 From: tommy100 at gmx.de (Tommy[D]) Date: Sat, 12 Jan 2008 00:31:56 +0100 Subject: [freenet-dev] [freenet-cvs] r16940 - trunk/freenet/src/freenet/l10n In-Reply-To: <200801111937.19777.toad@amphibian.dyndns.org> References: <20080106130844.6351147B849@freenetproject.org> <200801111937.19777.toad@amphibian.dyndns.org> Message-ID: <4787FC6C.80704@gmx.de> -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA512 Matthew Toseland schrieb: | On Sunday 06 January 2008 13:08, you wrote: |> Author: nextgens |> Date: 2008-01-06 13:08:43 +0000 (Sun, 06 Jan 2008) |> New Revision: 16940 |> |> Modified: |> trunk/freenet/src/freenet/l10n/freenet.l10n.de.properties |> trunk/freenet/src/freenet/l10n/freenet.l10n.es.properties |> trunk/freenet/src/freenet/l10n/freenet.l10n.it.properties |> Log: |> l10n: sed -i '/#freenet/d' *properties ... |> |> it\'s better to have an incomplete translation than an outdated one | | Are these going to be updated? Do we have maintainers for these languages? de is maintained (most by NEOatNHNG, some by me) -----BEGIN PGP SIGNATURE----- Version: GnuPG v2.0.8 (GNU/Linux) Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org iJwEAQEKAAYFAkeH/GwACgkQG7kqcTWJkGdD8gP9E3ZYOyeoNP00F/QUzSXv+kZV fH7F1pRmxwja4ujvNShGVKCFzFDGOmvhnmnrgONf1Oz+rODaRRjygV5ewSgufCbx M6FSCJtLujkkTPED3AAirH7ek3ERCtj0NsvBla7pfklyI3O+yYzd3Zi+rnAvJtLe FLtvKPZCaIEJEleMG/8= =0QZC -----END PGP SIGNATURE----- From robert at emu.freenetproject.org Fri Jan 11 23:50:20 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 17:50:20 -0600 Subject: [freenet-dev] Eureka! - r17024 - trunk/freenet/src/freenet/node In-Reply-To: <20080111234509.4400139085A@freenetproject.org> References: <20080111234509.4400139085A@freenetproject.org> Message-ID: I had just translated this bug to the prioritized queue... I kept the logic the same, but I knew it didn't look right! All messages requeued in the past before r17024 (which is to say, a lot of messages, because nodes are busy), are requeued in the reverse order. Statistically, they may be requeued again (and flipped back). With this bug gone (plus the priority queue) the insanely-long-request- searching has vanished! -- Robert Hailey On Jan 11, 2008, at 5:45 PM, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-11 23:45:09 +0000 (Fri, 11 Jan 2008) > New Revision: 17024 > > Modified: > trunk/freenet/src/freenet/node/PeerNode.java > Log: > requeue messages in the correct order... *long-standing-bug* > > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > =================================================================== > --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-11 23:22:03 > UTC (rev 17023) > +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-11 23:45:09 > UTC (rev 17024) > @@ -999,6 +999,25 @@ > } > > /** > + * like enqueuePrioritizedMessageItem, but adds it to the front of > those in the same priority. > + */ > + private void pushfrontPrioritizedMessageItem(MessageItem addMe) { > + synchronized (messagesToSendNow) { > + //Assume it goes on the end, both the common case and an > accelerator for requeueing. > + ListIterator > i=messagesToSendNow.listIterator(messagesToSendNow.size()); > + while (i.hasPrevious()) { > + MessageItem here=(MessageItem)i.previous(); > + //While the item we are adding is NOT-LESS-THAN priority, move > on (backwards...) > + if (!(addMe.getPriority() <= here.getPriority())) { > + i.next(); > + break; > + } > + } > + i.add(addMe); > + } > + } > + > + /** > * Returns the number of milliseconds that it is estimated to take > to transmit the currently queued packets. > */ > public long getProbableSendQueueTime() { > @@ -1186,7 +1205,7 @@ > synchronized(messagesToSendNow) { > for(int i = offset; i < offset + length; i++) > if(messages[i] != null) > - enqueuePrioritizedMessageItem(messages[i]); > + pushfrontPrioritizedMessageItem(messages[i]); > } > } > > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > From robert at emu.freenetproject.org Sat Jan 12 00:20:19 2008 From: robert at emu.freenetproject.org (Robert Hailey) Date: Fri, 11 Jan 2008 18:20:19 -0600 Subject: [freenet-dev] Eureka! - r17024 - trunk/freenet/src/freenet/node In-Reply-To: References: <20080111234509.4400139085A@freenetproject.org> Message-ID: <8A6AA370-6075-4FB6-8788-703601B142CC@emu.freenetproject.org> On Jan 11, 2008, at 5:50 PM, Robert Hailey wrote: > > I had just translated this bug to the prioritized queue... I kept the > logic the same, but I knew it didn't look right! > > All messages requeued in the past before r17024 (which is to say, a > lot of messages, because nodes are busy), are requeued in the reverse > order. Statistically, they may be requeued again (and flipped back). > > With this bug gone (plus the priority queue) the insanely-long- > request- > searching has vanished! > > -- > Robert Hailey I'm excited. If the speed of the simulator after r17025 is any indication of the overall freenet performance, wow... Inserts dropped from 28 seconds to 8 seconds (x3.5). Can't really measure requests, though, as they are already so fast in the simulator; but now they don't randomly hang of course. -- Robert Hailey From batosai at batosai.net Sat Jan 12 17:33:53 2008 From: batosai at batosai.net (Julien Cornuwel) Date: Sat, 12 Jan 2008 18:33:53 +0100 Subject: [freenet-dev] New PGP key Message-ID: <4788FA01.80603@batosai.net> Hi, Nextgens told me that my PGP key had been revoked. :-/ I had been thinking about revoking my permanent key, a few month ago, to replace it by a temporary one. When I updated my public keys on the pgp server today, the revokation must have been sent :( So, I'm forced to generate a new one with the old one already revoked. Sorry for that. Regards -------------- next part -------------- A non-text attachment was scrubbed... Name: 0x61917D90.asc Type: application/pgp-keys Size: 1700 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/0d5eb88e/attachment.key -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 250 bytes Desc: OpenPGP digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/0d5eb88e/attachment.pgp From toad at amphibian.dyndns.org Sat Jan 12 18:36:34 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Sat, 12 Jan 2008 18:36:34 +0000 Subject: [freenet-dev] Message priorities (and coalescing timeouts) In-Reply-To: <97CF0B6D-2F1A-414F-928A-A52B6BBEABCF@emu.freenetproject.org> References: <20080107204520.4EAB139230A@freenetproject.org> <200801111818.59091.toad@amphibian.dyndns.org> <97CF0B6D-2F1A-414F-928A-A52B6BBEABCF@emu.freenetproject.org> Message-ID: <200801121836.35265.toad@amphibian.dyndns.org> On Friday 11 January 2008 21:22, Robert Hailey wrote: > > On Jan 11, 2008, at 12:18 PM, Matthew Toseland wrote: > > > Are you interested in implementing message priorities? MessageItem and > > PacketSender are the most relevant classes. > > I'll certainly do the grunt work of putting in the priority constants > and accessors in (r17019), but I kinda don't want to touch > FNPPacketMangler/PacketSender. It's really not hard (look at the middle of PacketSender.run(), where it calculates the deadlines), but if you don't want to do it I will. > > Also, the packet acknowledgements appear to be a lower level than the > message queue. True. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/b242a12e/attachment.pgp From toad at amphibian.dyndns.org Sat Jan 12 18:37:25 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Sat, 12 Jan 2008 18:37:25 +0000 Subject: [freenet-dev] Message priorities (and coalescing timeouts) was Re: [freenet-cvs] r16960 - trunk/freenet/src/freenet/node In-Reply-To: <9D4E3198-1A5A-4178-B371-0F489C9847D9@emu.freenetproject.org> References: <20080107204520.4EAB139230A@freenetproject.org> <200801111740.35908.toad@amphibian.dyndns.org> <9D4E3198-1A5A-4178-B371-0F489C9847D9@emu.freenetproject.org> Message-ID: <200801121837.25877.toad@amphibian.dyndns.org> On Friday 11 January 2008 22:23, Robert Hailey wrote: > > On Jan 11, 2008, at 11:40 AM, Matthew Toseland wrote: > > > Group 1: > > 100ms coalescing timeout. > > Various messages with short timeouts, or which we need to send > > quickly for one > > reason or another. > > ... > > FNPPing/FNPPong - ???? > > FNPLinkPing/FNPLinkPong - ???? > > Good or bad, I don't know... but in order for [SUB_]MAX_PING_TIME to > be the same stat the pings would either have to be queued with the > bulk data or a separate data-level-ping be made. Presently I suppose > it gives the combined send queue lengths (to/from) + rtt (x2), and the > caps are 700ms & 1500ms. Yes and no. We don't use the above for ping time any more, we use the packet acknowledgements to calculate it. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/d492593d/attachment.pgp From toad at amphibian.dyndns.org Sat Jan 12 19:55:30 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Sat, 12 Jan 2008 19:55:30 +0000 Subject: [freenet-dev] Eureka! - r17024 - trunk/freenet/src/freenet/node In-Reply-To: References: <20080111234509.4400139085A@freenetproject.org> Message-ID: <200801121955.35227.toad@amphibian.dyndns.org> On Friday 11 January 2008 23:50, Robert Hailey wrote: > > I had just translated this bug to the prioritized queue... I kept the > logic the same, but I knew it didn't look right! > > All messages requeued in the past before r17024 (which is to say, a > lot of messages, because nodes are busy), are requeued in the reverse > order. Statistically, they may be requeued again (and flipped back). > > With this bug gone (plus the priority queue) the insanely-long-request- > searching has vanished! Cool! > > -- > Robert Hailey > > On Jan 11, 2008, at 5:45 PM, robert at freenetproject.org wrote: > > > Author: robert > > Date: 2008-01-11 23:45:09 +0000 (Fri, 11 Jan 2008) > > New Revision: 17024 > > > > Modified: > > trunk/freenet/src/freenet/node/PeerNode.java > > Log: > > requeue messages in the correct order... *long-standing-bug* > > > > > > Modified: trunk/freenet/src/freenet/node/PeerNode.java > > =================================================================== > > --- trunk/freenet/src/freenet/node/PeerNode.java 2008-01-11 23:22:03 > > UTC (rev 17023) > > +++ trunk/freenet/src/freenet/node/PeerNode.java 2008-01-11 23:45:09 > > UTC (rev 17024) > > @@ -999,6 +999,25 @@ > > } > > > > /** > > + * like enqueuePrioritizedMessageItem, but adds it to the front of > > those in the same priority. > > + */ > > + private void pushfrontPrioritizedMessageItem(MessageItem addMe) { > > + synchronized (messagesToSendNow) { > > + //Assume it goes on the end, both the common case and an > > accelerator for requeueing. > > + ListIterator > > i=messagesToSendNow.listIterator(messagesToSendNow.size()); > > + while (i.hasPrevious()) { > > + MessageItem here=(MessageItem)i.previous(); > > + //While the item we are adding is NOT-LESS-THAN priority, move > > on (backwards...) > > + if (!(addMe.getPriority() <= here.getPriority())) { > > + i.next(); > > + break; > > + } > > + } > > + i.add(addMe); > > + } > > + } > > + > > + /** > > * Returns the number of milliseconds that it is estimated to take > > to transmit the currently queued packets. > > */ > > public long getProbableSendQueueTime() { > > @@ -1186,7 +1205,7 @@ > > synchronized(messagesToSendNow) { > > for(int i = offset; i < offset + length; i++) > > if(messages[i] != null) > > - enqueuePrioritizedMessageItem(messages[i]); > > + pushfrontPrioritizedMessageItem(messages[i]); > > } > > } -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/08ae3c16/attachment.pgp From toad at amphibian.dyndns.org Sat Jan 12 19:59:45 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Sat, 12 Jan 2008 19:59:45 +0000 Subject: [freenet-dev] Eureka! - r17024 - trunk/freenet/src/freenet/node In-Reply-To: <8A6AA370-6075-4FB6-8788-703601B142CC@emu.freenetproject.org> References: <20080111234509.4400139085A@freenetproject.org> <8A6AA370-6075-4FB6-8788-703601B142CC@emu.freenetproject.org> Message-ID: <200801121959.45849.toad@amphibian.dyndns.org> On Saturday 12 January 2008 00:20, Robert Hailey wrote: > > On Jan 11, 2008, at 5:50 PM, Robert Hailey wrote: > > > > > I had just translated this bug to the prioritized queue... I kept the > > logic the same, but I knew it didn't look right! > > > > All messages requeued in the past before r17024 (which is to say, a > > lot of messages, because nodes are busy), are requeued in the reverse > > order. Statistically, they may be requeued again (and flipped back). > > > > With this bug gone (plus the priority queue) the insanely-long- > > request- > > searching has vanished! > > > > -- > > Robert Hailey > > > I'm excited. If the speed of the simulator after r17025 is any > indication of the overall freenet performance, wow... > > Inserts dropped from 28 seconds to 8 seconds (x3.5). Can't really > measure requests, though, as they are already so fast in the > simulator; but now they don't randomly hang of course. Fascinating. I'm surprised that it had such a strong effect - how much of this is due to prioritising and how much due to queue reversal? With prioritisation I'd expect the non-bulk queues to get flushed fairly quickly, no? -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/8823b1cf/attachment.pgp From toad at amphibian.dyndns.org Sat Jan 12 20:01:57 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Sat, 12 Jan 2008 20:01:57 +0000 Subject: [freenet-dev] [freenet-cvs] r16987 - trunk/freenet/src/freenet/io/xfer In-Reply-To: <79F840DD-6D9C-4515-9214-B8B9F6318FBE@emu.freenetproject.org> References: <20080109221037.8B642479874@freenetproject.org> <200801112206.24857.toad@amphibian.dyndns.org> <79F840DD-6D9C-4515-9214-B8B9F6318FBE@emu.freenetproject.org> Message-ID: <200801122001.57608.toad@amphibian.dyndns.org> On Friday 11 January 2008 22:46, Robert Hailey wrote: > > On Jan 11, 2008, at 4:06 PM, Matthew Toseland wrote: > > > On Wednesday 09 January 2008 22:10, robert at freenetproject.org wrote: > >> Author: robert > >> Date: 2008-01-09 22:10:36 +0000 (Wed, 09 Jan 2008) > >> New Revision: 16987 > >> > >> Modified: > >> trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > >> Log: > >> don't assume everything has been transmitted when the transmitter > >> oversleeps > >> - should get rid of "haven't heard from receiver in 1m55s." message > >> - the transmitter still oversleeps (2 minutes?!), and is not woken > >> up by > > notifyAll() because it is in the throttle function: delay() > > > > Why does the below help? > > I think it was actually because delay() was deadlocking, not > oversleeping. In any case, this helps then because we will not then > see an old timeAllSent and assume that the receiver is dead ("haven't > heard from the receiver"). I still don't understand this - timeAllSent isn't set until not only have we sent all the packets but we've also received all the packets. Therefore, packetReceived() won't be called after that, because packetReceived() means we've received a new packet from the previous sender. I can see that we'd want to reset it in some other cases, but not here. > > -- > Robert Hailey > > > > >> > >> > >> Modified: trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > >> =================================================================== > >> --- trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > >> 2008-01-09 > > 18:33:46 UTC (rev 16986) > >> +++ trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > >> 2008-01-09 > > 22:10:36 UTC (rev 16987) > >> @@ -192,6 +192,7 @@ > >> public void packetReceived(int packetNo) { > >> synchronized(_senderThread) { > >> _unsent.addLast(new Integer(packetNo)); > >> + timeAllSent = -1; > >> _sentPackets.setBit(packetNo, false); > >> _senderThread.notifyAll(); > >> } > > _______________________________________________ > > Devl mailing list > > Devl at freenetproject.org > > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/dd9dd07d/attachment.pgp From toad at amphibian.dyndns.org Sat Jan 12 20:04:24 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Sat, 12 Jan 2008 20:04:24 +0000 Subject: [freenet-dev] One tunnel per request: justification In-Reply-To: <4787D717.1020808@cs.ucl.ac.uk> References: <200712192256.08351.toad@amphibian.dyndns.org> <4787BE1B.8090508@cs.ucl.ac.uk> <4787D717.1020808@cs.ucl.ac.uk> Message-ID: <200801122004.25154.toad@amphibian.dyndns.org> On Friday 11 January 2008 20:52, Michael Rogers wrote: > Michael Rogers wrote: > > I have some notes somewhere on various ways of defining cluster and > > cliques... I'll see if there's anything that could be used to agree on > > the membership of a cell. > > Two approaches from Knoke & Kuklinski: > > 1) n-cliques: every member can reach every other in n hops. (We'd need > to decide whether paths may pass through non-members for the purposes of > calculating the membership.) > > 2) k-plex cliques: each of the n members is connected to at least n-k > others (so a fully connected subgraph is a 1-plex clique). > > But in either case I'm not sure how to work out which cliques a node > belongs to using only its local view of the network. In the k-plex case, > what if the clique is larger than my local view? In the n-clique case, > if two nodes within n hops of me aren't within n hops of each other, > which do I eliminate? Good question. I dunno. Can you put some of this on the wiki page on premix routing? The more urgent matter IMHO is fleshing out tunnels, which we could implement (as an option) in the comparatively near future, maybe even for 0.7. > > Cheers, > Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/0e747e92/attachment.pgp From toad at amphibian.dyndns.org Sat Jan 12 20:08:53 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Sat, 12 Jan 2008 20:08:53 +0000 Subject: [freenet-dev] [freenet-cvs] r16993 - trunk/freenet/src/freenet/io/xfer In-Reply-To: <20080110164027.93190479703@freenetproject.org> References: <20080110164027.93190479703@freenetproject.org> Message-ID: <200801122008.53245.toad@amphibian.dyndns.org> On Thursday 10 January 2008 16:40, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-10 16:40:27 +0000 (Thu, 10 Jan 2008) > New Revision: 16993 > > Modified: > trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > Log: > simplify _senderThread You've removed the resetting of timeAllSent when _unsent.size() == 0. The rest is reasonable refactoring. > > > Modified: trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > =================================================================== > --- trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java 2008-01-10 15:06:43 UTC (rev 16992) > +++ trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java 2008-01-10 16:40:27 UTC (rev 16993) > @@ -81,61 +81,50 @@ > public void run() { > while (!_sendComplete) { > long startCycleTime = System.currentTimeMillis(); > + int packetNo; > try { > - while (true) { > - synchronized(_senderThread) { > - if(_unsent.size() != 0) { > - timeAllSent = -1; > - break; > - } > - // No unsent packets > - if(getNumSent() == _prb.getNumPackets()) { > - //No unreceived packets > - if(Logger.shouldLog(Logger.MINOR, this)) > - Logger.minor(this, "Sent all blocks, none unsent"); > - if(timeAllSent <= 0) > - timeAllSent = System.currentTimeMillis(); > - } > + synchronized(_senderThread) { > + while (_unsent.size() == 0) { > if(_sendComplete) return; > _senderThread.wait(10*1000); > } > - } > - } catch (InterruptedException e) { > - } catch (AbortedException e) { > - synchronized(_senderThread) { > - _sendComplete = true; > - _senderThread.notifyAll(); > - } > - return; > - } > - int packetNo; > - try { > - synchronized(_senderThread) { > packetNo = ((Integer) _unsent.removeFirst()).intValue(); > } > - } catch (NoSuchElementException nsee) { > - // back up to the top to check for completion > + } catch (InterruptedException e) { > + Logger.error(this, "_senderThread interrupted"); > continue; > } > - delay(startCycleTime); > - if(_sendComplete) break; > - _sentPackets.setBit(packetNo, true); > + int totalPackets; > try { > _destination.sendAsync(DMT.createPacketTransmit(_uid, packetNo, _sentPackets, _prb.getPacket(packetNo)), null, PACKET_SIZE, _ctr); > _ctr.sentPayload(PACKET_SIZE); > + totalPackets=_prb.getNumPackets(); > } catch (NotConnectedException e) { > Logger.normal(this, "Terminating send: "+e); > synchronized(_senderThread) { > _sendComplete = true; > _senderThread.notifyAll(); > + return; > } > } catch (AbortedException e) { > Logger.normal(this, "Terminating send due to abort: "+e); > synchronized(_senderThread) { > _sendComplete = true; > _senderThread.notifyAll(); > + return; > } > } > + synchronized (_senderThread) { > + _sentPackets.setBit(packetNo, true); > + if(_unsent.size() == 0 && getNumSent() == totalPackets) { > + //No unsent packets, no unreceived packets > + timeAllSent = System.currentTimeMillis(); > + if(Logger.shouldLog(Logger.MINOR, this)) > + Logger.minor(this, "Sent all blocks, none unsent"); > + _senderThread.notifyAll(); > + } > + } > + delay(startCycleTime); > } > } > > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/fa03f076/attachment.pgp From toad at amphibian.dyndns.org Sat Jan 12 20:12:08 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Sat, 12 Jan 2008 20:12:08 +0000 Subject: [freenet-dev] [freenet-cvs] r16997 - in trunk/freenet/src/freenet/io: comm xfer In-Reply-To: <20080110192300.CEA9D47A17D@freenetproject.org> References: <20080110192300.CEA9D47A17D@freenetproject.org> Message-ID: <200801122012.08408.toad@amphibian.dyndns.org> On Thursday 10 January 2008 19:23, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-10 19:23:00 +0000 (Thu, 10 Jan 2008) > New Revision: 16997 > > Modified: > trunk/freenet/src/freenet/io/comm/RetrievalException.java > trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > Log: > tell the reciever if we are not transmitting anymore, and why > > > Modified: trunk/freenet/src/freenet/io/comm/RetrievalException.java > =================================================================== > --- trunk/freenet/src/freenet/io/comm/RetrievalException.java 2008-01-10 19:19:20 UTC (rev 16996) > +++ trunk/freenet/src/freenet/io/comm/RetrievalException.java 2008-01-10 19:23:00 UTC (rev 16997) > @@ -40,6 +40,7 @@ > public static final int NO_DATAINSERT = 8; > public static final int CANCELLED_BY_RECEIVER = 9; > public static final int RANGE_UNSUPPORTED = 10; //was 4 > + public static final int RECEIVER_DIED = 11; > > int _reason; > String _cause; > @@ -50,7 +51,7 @@ > } > > public RetrievalException(int reason, String cause) { > - this(reason); > + _reason = reason; > _cause = cause; > if (cause==null || cause.length()==0 || cause.equals("null")) > _cause=getErrString(reason); > > Modified: trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java > =================================================================== > --- trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java 2008-01-10 19:19:20 UTC (rev 16996) > +++ trunk/freenet/src/freenet/io/xfer/BlockTransmitter.java 2008-01-10 19:23:00 UTC (rev 16997) > @@ -30,6 +30,7 @@ > import freenet.io.comm.MessageFilter; > import freenet.io.comm.NotConnectedException; > import freenet.io.comm.PeerContext; > +import freenet.io.comm.RetrievalException; > import freenet.node.PeerNode; > import freenet.support.BitArray; > import freenet.support.DoubleTokenBucket; > @@ -208,6 +209,7 @@ > _sendComplete = true; > _senderThread.notifyAll(); > } > + sendAborted(_prb.getAbortedReason(), "Downstream transfer failed"); > return false; > } > Message msg; > @@ -219,20 +221,20 @@ > msg = _usm.waitFor(mfMissingPacketNotification.or(mfAllReceived.or(mfSendAborted)), _ctr); > if(logMINOR) Logger.minor(this, "Got "+msg); > } catch (DisconnectedException e) { > - // Ignore, see below > - msg = null; > - } > - if(logMINOR) Logger.minor(this, "Got "+msg); > - if(!_destination.isConnected()) { > Logger.normal(this, "Terminating send "+_uid+" to "+_destination+" from "+_destination.getSocketHandler()+" because node disconnected while waiting"); > synchronized(_senderThread) { > _sendComplete = true; > _senderThread.notifyAll(); > } > + //They disconnected, can't send an abort to them then can we? > return false; > } Is this part (above) an improvement? > - if(_sendComplete) > + if(logMINOR) Logger.minor(this, "Got "+msg); > + if(_sendComplete) { > + Logger.normal(this, "send cancelled by _senderThread"); > + //_senderThread will have sent an aborted message > return false; > + } > if (msg == null) { > long now = System.currentTimeMillis(); > //SEND_TIMEOUT (one minute) after all packets have been transmitted, terminate the send. > @@ -242,7 +244,9 @@ > _sendComplete = true; > _senderThread.notifyAll(); > } > - Logger.error(this, "Terminating send "+_uid+" to "+_destination+" from "+_destination.getSocketHandler()+" as we haven't heard from receiver in "+TimeUtil.formatTime((now - timeAllSent), 2, true)+ '.'); > + String timeString=TimeUtil.formatTime((now - timeAllSent), 2, true); > + Logger.error(this, "Terminating send "+_uid+" to "+_destination+" from "+_destination.getSocketHandler()+" as we haven't heard from receiver in "+timeString+ '.'); > + sendAborted(RetrievalException.RECEIVER_DIED, "Haven't heard from you (receiver) in "+timeString); > return false; > } else { > if(logMINOR) Logger.minor(this, "Ignoring timeout: timeAllSent="+timeAllSent+" ("+(System.currentTimeMillis() - timeAllSent)+"), getNumSent="+getNumSent()+ '/' +_prb.getNumPackets()); > @@ -276,18 +280,30 @@ > _sendComplete = true; > _senderThread.notifyAll(); > } > + //They aborted, don't need to send an aborted back :) > return false; > } else { > Logger.error(this, "Transmitter received unknown message type: "+msg.getSpec().getName()); > } > } > + } catch (NotConnectedException e) { > + //most likely from sending an abort() > + Logger.normal(this, "NotConnectedException in BlockTransfer.send():"+e); > + return false; > } catch (AbortedException e) { > - // Terminate > + Logger.normal(this, "AbortedException in BlockTransfer.send():"+e); > + try { > + sendAborted(RetrievalException.CANCELLED_BY_RECEIVER, "Downstream transfer failed"); > + } catch (NotConnectedException gone) { > + //ignore > + } > + return false; > + } finally { > + //Terminate, if we are not listening for control packets, don't be sending any data > synchronized(_senderThread) { > _sendComplete = true; > _senderThread.notifyAll(); > - } > - return false; > + } > } > } > > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > > -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20080112/739c2735/attachment.pgp From toad at amphibian.dyndns.org Sat Jan 12 20:29:24 2008 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Sat, 12 Jan 2008 20:29:24 +0000 Subject: [freenet-dev] [freenet-cvs] r17019 - trunk/freenet/src/freenet/io/comm In-Reply-To: <20080111212036.82C1C3922B9@freenetproject.org> References: <20080111212036.82C1C3922B9@freenetproject.org> Message-ID: <200801122029.29531.toad@amphibian.dyndns.org> Maybe we should have a longer timeout on SSKInsertRequest? It's as big as a bulk datum, but it's done in a regular request, with (presumably) a short timeout ... I don't like the idea of making it stick out on traffic analysis, but I don't like the idea of it timing out more often than other requests either... We can either move it up to HIGH, or we can have a very long timeout on SSK inserts?? On Friday 11 January 2008 21:20, robert at freenetproject.org wrote: > Author: robert > Date: 2008-01-11 21:20:36 +0000 (Fri, 11 Jan 2008) > New Revision: 17019 > > Modified: > trunk/freenet/src/freenet/io/comm/DMT.java > trunk/freenet/src/freenet/io/comm/MessageType.java > Log: > enter initial message priorities & an accessor therefor > > > Modified: trunk/freenet/src/freenet/io/comm/DMT.java > =================================================================== > --- trunk/freenet/src/freenet/io/comm/DMT.java 2008-01-11 20:49:57 UTC (rev 17018) > +++ trunk/freenet/src/freenet/io/comm/DMT.java 2008-01-11 21:20:36 UTC (rev 17019) > @@ -126,9 +126,16 @@ > public static final String REJECT_CODE = "rejectCode"; > public static final String ROUTING_ENABLED = "routingEnabled"; > > + public static final short PRIORITY_NOW=-2; > + public static final short PRIORITY_HIGH=-1; > + public static final short PRIORITY_UNSPECIFIED=0; > + public static final short PRIORITY_LOW=1; > + public static final short PRIORITY_BULK_DATA=2; > + > // Assimilation > + > // New data transmission messages > - public static final MessageType packetTransmit = new MessageType("packetTransmit") {{ > + public static final MessageType packetTransmit = new MessageType("packetTransmit", PRIORITY_BULK_DATA) {{ > addField(UID, Long.class); > addField(PACKET_NO, Integer.class); > addField(SENT, BitArray.class); > @@ -153,7 +160,7 @@ > return size + 8 /* uid */ + 4 /* packet# */ + 4 /* Message hader */; > } > > - public static final MessageType allSent = new MessageType("allSent") {{ > + public static final MessageType allSent = new MessageType("allSent", PRIORITY_LOW) {{ > addField(UID, Long.class); > }}; > > @@ -163,7 +170,7 @@ > return msg; > } > > - public static final MessageType missingPacketNotification = new MessageType("missingPacketNotification") {{ > + public static final MessageType missingPacketNotification = new MessageType("missingPacketNotification", PRIORITY_LOW) {{ > addField(UID, Long.class); > addLinkedListField(MISSING, Integer.class); > }}; > @@ -175,7 +182,7 @@ > return msg; > } > > - public static final MessageType allReceived = new MessageType("allReceived") {{ > + public static final MessageType allReceived = new MessageType("allReceived", PRIORITY_LOW) {{ > addField(UID, Long.class); > }}; > public static final Message createAllReceived(long uid) { > @@ -184,7 +191,7 @@ > return msg; > } > > - public static final MessageType sendAborted = new MessageType("sendAborted") {{ > + public static final MessageType sendAborted = new MessageType("sendAborted", PRIORITY_LOW) {{ > addField(UID, Long.class); > addField(DESCRIPTION, String.class); > addField(REASON, Integer.class); > @@ -198,7 +205,7 @@ > return msg; > } > > - public static final MessageType FNPBulkPacketSend = new MessageType("FNPBulkPacketSend") {{ > + public static final MessageType FNPBulkPacketSend = new MessageType("FNPBulkPacketSend", PRIORITY_BULK_DATA) {{ > addField(UID, Long.class); > addField(PACKET_NO, Integer.class); > addField(DATA, ShortBuffer.class); > @@ -216,7 +223,7 @@ > return createFNPBulkPacketSend(uid, packetNo, new ShortBuffer(data)); > } > > - public static final MessageType FNPBulkSendAborted = new MessageType("FNPBulkSendAborted") {{ > + public static final MessageType FNPBulkSendAborted = new MessageType("FNPBulkSendAborted", PRIORITY_LOW) {{ > addField(UID, Long.class); > }}; > > @@ -226,7 +233,7 @@ > return msg; > } > > - public static final MessageType FNPBulkReceiveAborted = new MessageType("FNPBulkReceiveAborted") {{ > + public static final MessageType FNPBulkReceiveAborted = new MessageType("FNPBulkReceiveAborted", PRIORITY_LOW) {{ > addField(UID, Long.class); > }}; > > @@ -236,7 +243,7 @@ > return msg; > } > > - public static final MessageType FNPBulkReceivedAll = new MessageType("FNPBulkReceivedAll") {{ > + public static final MessageType FNPBulkReceivedAll = new MessageType("FNPBulkReceivedAll", PRIORITY_LOW) {{ > addField(UID, Long.class); > }}; > > @@ -246,7 +253,7 @@ > return msg; > } > > - public static final MessageType testTransferSend = new MessageType("testTransferSend") {{ > + public static final MessageType testTransferSend = new MessageType("testTransferSend", PRIORITY_UNSPECIFIED) {{ > addField(UID, Long.class); > }}; > > @@ -256,7 +263,7 @@ > return msg; > } > > - public static final MessageType testTransferSendAck = new MessageType("testTransferSendAck") {{ > + public static final MessageType testTransferSendAck = new MessageType("testTransferSendAck", PRIORITY_UNSPECIFIED) {{ > addField(UID, Long.class); > }}; > > @@ -266,7 +273,7 @@ > return msg; > } > > - public static final MessageType testSendCHK = new MessageType("testSendCHK") {{ > + public static final MessageType testSendCHK = new MessageType("testSendCHK", PRIORITY_UNSPECIFIED) {{ > addField(UID, Long.class); > addField(FREENET_URI, String.class); > addField(CHK_HEADER, Buffer.class); > @@ -280,7 +287,7 @@ > return msg; > } > > - public static final MessageType testRequest = new MessageType("testRequest") {{ > + public static final MessageType testRequest = new MessageType("testRequest", PRIORITY_UNSPECIFIED) {{ > addField(UID, Long.class); > addField(FREENET_ROUTING_KEY, Key.class); > addField(HTL, Integer.class); > @@ -294,7 +301,7 @@ > return msg; > } > > - public static final MessageType testDataNotFound = new MessageType("testDataNotFound") {{ > + public static final MessageType testDataNotFound = new MessageType("testDataNotFound", PRIORITY_UNSPECIFIED) {{ > addField(UID, Long.class); > }}; > > @@ -304,7 +311,7 @@ > return msg; > } > > - public static final MessageType testDataReply = new MessageType("testDataReply") {{ > + public static final MessageType testDataReply = new MessageType("testDataReply", PRIORITY_UNSPECIFIED) {{ > addField(UID, Long.class); > addField(TEST_CHK_HEADERS, Buffer.class); > }}; > @@ -316,7 +323,7 @@ > return msg; > } > > - public static final MessageType testSendCHKAck = new MessageType("testSendCHKAck") {{ > + public static final MessageType testSendCHKAck = new MessageType("testSendCHKAck", PRIORITY_UNSPECIFIED) {{ > addField(UID, Long.class); > addField(FREENET_URI, String.class); > }}; > @@ -327,7 +334,7 @@ > return msg; > } > > - public static final MessageType testDataReplyAck = new MessageType("testDataReplyAck") {{ > + public static final MessageType testDataReplyAck = new MessageType("testDataReplyAck", PRIORITY_UNSPECIFIED) {{ > addField(UID, Long.class); > }}; > > @@ -337,7 +344,7 @@ > return msg; > } > > - public static final MessageType testDataNotFoundAck = new MessageType("testDataNotFoundAck") {{ > + public static final MessageType testDataNotFoundAck = new MessageType("testDataNotFoundAck", PRIORITY_UNSPECIFIED) {{ > addField(UID, Long.class); > }}; > public static final Message createTestDataNotFoundAck(long id) { > @@ -348,7 +355,7 @@ > > // Internal only messages > > - public static final MessageType testReceiveCompleted = new MessageType("testReceiveCompleted", true) {{ > + public static final MessageType testReceiveCompleted = new MessageType("testReceiveCompleted", PRIORITY_UNSPECIFIED, true) {{ > addField(UID, Long.class); > addField(SUCCESS, Boolean.class); > addField(REASON, String.class); > @@ -362,7 +369,7 @@ > return msg; > } > > - public static final MessageType testSendCompleted = new MessageType("testSendCompleted", true) {{ > + public static final MessageType testSendCompleted = new MessageType("testSendCompleted", PRIORITY_UNSPECIFIED, true) {{ > addField(UID, Long.class); > addField(SUCCESS, Boolean.class); > addField(REASON, String.class); > @@ -377,7 +384,7 @@ > } > > // Node-To-Node Message (generic) > - public static final MessageType nodeToNodeMessage = new MessageType("nodeToNodeMessage", false) {{ > + public static final MessageType nodeToNodeMessage = new MessageType("nodeToNodeMessage", PRIORITY_LOW, false) {{ > addField(NODE_TO_NODE_MESSAGE_TYPE, Integer.class); > addField(NODE_TO_NODE_MESSAGE_DATA, ShortBuffer.class); > }}; > @@ -390,7 +397,7 @@ > } > > // FNP messages > - public static final MessageType FNPCHKDataRequest = new MessageType("FNPCHKDataRequest") {{ > + public static final MessageType FNPCHKDataRequest = new MessageType("FNPCHKDataRequest", PRIORITY_HIGH) {{ > addField(UID, Long.class); > addField(HTL, Short.class); > addField(NEAREST_LOCATION, Double.class); > @@ -406,7 +413,7 @@ > return msg; > } > > - public static final MessageType FNPSSKDataRequest = new MessageType("FNPSSKDataRequest") {{ > + public static final MessageType FNPSSKDataRequest = new MessageType("FNPSSKDataRequest", PRIORITY_HIGH) {{ > addField(UID, Long.class); > addField(HTL, Short.class); > addField(NEAREST_LOCATION, Double.class); > @@ -425,7 +432,7 @@ > } > > // Hit our tail, try a different node. > - public static final MessageType FNPRejectedLoop = new MessageType("FNPRejectLoop") {{ > + public static final MessageType FNPRejectedLoop = new MessageType("FNPRejectLoop", PRIORITY_HIGH) {{ > addField(UID, Long.class); > }}; > > @@ -437,7 +444,7 @@ > > // Too many requests for present capacity. Fail, propagate back > // to source, and reduce send rate. > - public static final MessageType FNPRejectedOverload = new MessageType("FNPRejectOverload") {{ > + public static final MessageType FNPRejectedOverload = new MessageType("FNPRejectOverload", PRIORITY_HIGH) {{ > addField(UID, Long.class); > addField(IS_LOCAL, Boolean.class); > }}; > @@ -449,7 +456,7 @@ > return msg; > } > > - public static final MessageType FNPAccepted = new MessageType("FNPAccepted") {{ > + public static final MessageType FNPAccepted = new MessageType("FNPAccepted", PRIORITY_HIGH) {{ > addField(UID, Long.class); > }}; > > @@ -459,7 +466,7 @@ > return msg; > } > > - public static final MessageType FNPDataNotFound = new MessageType("FNPDataNotFound") {{ > + public static final MessageType FNPDataNotFound = new MessageType("FNPDataNotFound", PRIORITY_LOW) {{ > addField(UID, Long.class); > }}; > > @@ -469,7 +476,7 @@ > return msg; > } > > - public static final MessageType FNPRecentlyFailed = new MessageType("FNPRecentlyFailed") {{ > + public static final MessageType FNPRecentlyFailed = new MessageType("FNPRecentlyFailed", PRIORITY_HIGH) {{ > addField(UID, Long.class); > addField(TIME_LEFT, Integer.class); > }}; > @@ -481,7 +488,7 @@ > return msg; > } > > - public static final MessageType FNPCHKDataFound = new MessageType("FNPCHKDataFound") {{ > + public static final MessageType FNPCHKDataFound = new MessageType("FNPCHKDataFound", PRIORITY_LOW) {{ > addField(UID, Long.class); > addField(BLOCK_HEADERS, ShortBuffer.class); > }}; > @@ -493,7 +500,7 @@ > return msg; > } > > - public static final MessageType FNPRouteNotFound = new MessageType("FNPRouteNotFound") {{ > + public static final MessageType FNPRouteNotFound = new MessageType("FNPRouteNotFound", PRIORITY_LOW) {{ > addField(UID, Long.class); > addField(HTL, Short.class); > }}; > @@ -505,7 +512,7 @@ > return msg; > } > > - public static final MessageType FNPInsertRequest = new MessageType("FNPInsertRequest") {{ > + public static final MessageType FNPInsertRequest = new MessageType("FNPInsertRequest", PRIORITY_HIGH) {{ > addField(UID, Long.class); > addField(HTL, Short.class); > addField(NEAREST_LOCATION, Double.class); > @@ -521,7 +528,7 @@ > return msg; > } > > - public static final MessageType FNPInsertReply = new MessageType("FNPInsertReply") {{ > + public static final MessageType FNPInsertReply = new MessageType("FNPInsertReply", PRIORITY_LOW) {{ > addField(UID, Long.class); > }}; > > @@ -531,7 +538,7 @@ > return msg; > } > > - public static final MessageType FNPDataInsert = new MessageType("FNPDataInsert") {{ > + public static final MessageType FNPDataInsert = new MessageType("FNPDataInsert", PRIORITY_HIGH) {{ > addField(UID, Long.class); > addField(BLOCK_HEADERS, ShortBuffer.class); > }}; > @@ -543,7 +550,7 @@ > return msg; > } > > - public static final MessageType FNPInsertTransfersCompleted = new MessageType("FNPInsertTransfersCompleted") {{ > + public static final MessageType FNPInsertTransfersCompleted = new MessageType("FNPInsertTransfersCompleted", PRIORITY_LOW) {{ > addField(UID, Long.class); > addField(ANY_TIMED_OUT, Boolean.class); > }}; > @@ -555,7 +562,7 @@ > return msg; > } > > - public static final MessageType FNPRejectedTimeout = new MessageType("FNPTooSlow") {{ > + public static final MessageType FNPRejectedTimeout = new MessageType("FNPTooSlow", PRIORITY_LOW) {{ > addField(UID, Long.class); > }}; > > @@ -565,7 +572,7 @@ > return msg; > } > > - public static final MessageType FNPDataInsertRejected = new MessageType("FNPDataInsertRejected") {{ > + public static final MessageType FNPDataInsertRejected = new MessageType("FNPDataInsertRejected", PRIORITY_LOW) {{ > addField(UID, Long.class); > addField(DATA_INSERT_REJECTED_REASON, Short.class); > }}; > @@ -591,7 +598,7 @@ > return "Unknown reason code: "+reason; > } > > - public static final MessageType FNPSSKInsertRequest = new MessageType("FNPSSKInsertRequest") {{ > + public static final MessageType FNPSSKInsertRequest = new MessageType("FNPSSKInsertRequest", PRIORITY_BULK_DATA) {{ > addField(UID, Long.class); > addField(HTL, Short.class); > addField(FREENET_ROUTING_KEY, NodeSSK.class); > @@ -613,7 +620,7 @@ > return msg; > } > > - public static final MessageType FNPSSKDataFound = new MessageType("FNPSSKDataFound") {{ > + public static final MessageType FNPSSKDataFound = new MessageType("FNPSSKDataFound", PRIORITY_BULK_DATA) {{ > addField(UID, Long.class); > addField(BLOCK_HEADERS, ShortBuffer.class); > addField(DATA, ShortBuffer.class); > @@ -627,7 +634,7 @@ > return msg; > } > > - public static MessageType FNPSSKAccepted = new MessageType("FNPSSKAccepted") {{ > + public static MessageType FNPSSKAccepted = new MessageType("FNPSSKAccepted", PRIORITY_HIGH) {{ > addField(UID, Long.class); > addField(NEED_PUB_KEY, Boolean.class); > }}; > @@ -639,7 +646,7 @@ > return msg; > } > > - public static MessageType FNPSSKPubKey = new MessageType("FNPSSKPubKey") {{ > + public static MessageTy