From juiceman69 at gmail.com Fri Jun 1 05:00:44 2007 From: juiceman69 at gmail.com (Juiceman) Date: Fri, 1 Jun 2007 01:00:44 -0400 Subject: [freenet-dev] [freenet-cvs] r13423 - trunk/freenet/src/freenet/io/comm In-Reply-To: <20070531212049.1DB3E47983B@emu.freenetproject.org> References: <20070531212049.1DB3E47983B@emu.freenetproject.org> Message-ID: <8b525dee0705312200y4e19a9bfmea4d242e1cdacc40@mail.gmail.com> On 5/31/07, toad at freenetproject.org wrote: > Author: toad > Date: 2007-05-31 21:20:48 +0000 (Thu, 31 May 2007) > New Revision: 13423 > > Modified: > trunk/freenet/src/freenet/io/comm/MessageFilter.java > trunk/freenet/src/freenet/io/comm/UdpSocketManager.java > Log: > Refactoring on MessageFilter/USM. Reduces locking a bit. Will introduce asynchronous MessageFilter API soon. > > Modified: trunk/freenet/src/freenet/io/comm/MessageFilter.java > =================================================================== > --- trunk/freenet/src/freenet/io/comm/MessageFilter.java 2007-05-31 20:50:31 UTC (rev 13422) > +++ trunk/freenet/src/freenet/io/comm/MessageFilter.java 2007-05-31 21:20:48 UTC (rev 13423) > @@ -200,4 +200,19 @@ > _droppedConnection = ctx; > } > } > + > + /** > + * Notify waiters that we have been matched. > + * Hopefully no locks will be held at this point by the caller. > + */ > + public synchronized void onMatched() { > + notifyAll(); > + } > + > + /** > + * Notify waiters that we have timed out. > + */ > + public synchronized void onTimedOut() { > + notifyAll(); > + } > } > > Modified: trunk/freenet/src/freenet/io/comm/UdpSocketManager.java > =================================================================== > --- trunk/freenet/src/freenet/io/comm/UdpSocketManager.java 2007-05-31 20:50:31 UTC (rev 13422) > +++ trunk/freenet/src/freenet/io/comm/UdpSocketManager.java 2007-05-31 21:20:48 UTC (rev 13423) > @@ -155,6 +155,7 @@ > // } > // Only used for debugging, no need to seed from Yarrow > dropRandom = new Random(); > + _timedOutFilters = new Vector(32); > logMINOR = Logger.shouldLog(Logger.MINOR, this); > } > > @@ -294,17 +295,24 @@ > return true; > } > > + /** Only used by removeTimedOutFilters() - if future code uses this elsewhere, we need to > + * reconsider its locking. */ > + private final Vector _timedOutFilters; > + > + /** > + * Remove timed out filters. > + * Only called by realRun(), so it can move timed out filters to the _timedOutFilters array, > + * and then tell them that they are timed out without holding locks. > + * > + */ > private void removeTimedOutFilters() { > long tStart = System.currentTimeMillis(); > synchronized (_filters) { > for (ListIterator i = _filters.listIterator(); i.hasNext();) { > MessageFilter f = (MessageFilter) i.next(); > if (f.timedOut()) { > - f.setMessage(null); > - synchronized (f) { > - i.remove(); > - f.notifyAll(); > - } > + i.remove(); > + _timedOutFilters.add(f); > } else { // Because _filters are in order of timeout, we > // can abort the iteration as soon as we find one that > // doesn't timeout > @@ -312,6 +320,17 @@ > } > } > } > + > + for(int i=0;i<_timedOutFilters.size();i++) { > + MessageFilter f = (MessageFilter) _timedOutFilters.get(i); > + f.setMessage(null); > + f.onTimedOut(); > + synchronized (f) { > + f.notifyAll(); > + } > + } > + _timedOutFilters.clear(); > + > long tEnd = System.currentTimeMillis(); > if(tEnd - tStart > 50) { > if(tEnd - tStart > 3000) > @@ -343,14 +362,15 @@ > + m.getSource() + " : " + m); > } > } > + MessageFilter match = null; > synchronized (_filters) { > for (ListIterator i = _filters.listIterator(); i.hasNext();) { > MessageFilter f = (MessageFilter) i.next(); > if (f.match(m)) { > matched = true; > - f.setMessage(m); > + i.remove(); > + match = f; > synchronized (f) { > - i.remove(); > f.notifyAll(); > } > if(logMINOR) Logger.minor(this, "Matched: "+f); > @@ -358,6 +378,8 @@ > } > } > } > + match.setMessage(m); > + match.onMatched(); > // Feed unmatched messages to the dispatcher > if ((!matched) && (_dispatcher != null)) { > try { > @@ -397,11 +419,8 @@ > MessageFilter f = (MessageFilter) i.next(); > if (f.match(m)) { > matched = true; > - f.setMessage(m); > - synchronized (f) { > - i.remove(); > - f.notifyAll(); > - } > + match = f; > + i.remove(); > if(logMINOR) Logger.minor(this, "Matched: "+f); > break; // Only one match permitted per message > } > @@ -420,6 +439,10 @@ > if(logMINOR) Logger.minor(this, "Done"); > } > } > + if(match != null) { > + match.setMessage(m); > + match.onMatched(); > + } > } > long tEnd = System.currentTimeMillis(); > if(tEnd - tStart > 50) { > This gives me a NPE Jun 01, 2007 04:25:01:906 (freenet.io.comm.UdpSocketManager, UdpSocketManager packet receiver thread on port 10692, ERROR): Caught java.lang.NullPointerException from freenet.node.FNPPacketMangler at 75d758 java.lang.NullPointerException at freenet.io.comm.UdpSocketManager.checkFilters(UdpSocketManager.java:381) at freenet.node.FNPPacketMangler.processDecryptedData(FNPPacketMangler.java:890) at freenet.node.FNPPacketMangler.tryProcess(FNPPacketMangler.java:718) at freenet.node.FNPPacketMangler.process(FNPPacketMangler.java:118) at freenet.io.comm.UdpSocketManager.realRun(UdpSocketManager.java:245) at freenet.io.comm.UdpSocketManager.runLoop(UdpSocketManager.java:212) at freenet.io.comm.UdpSocketManager.run(UdpSocketManager.java:168) From toad at amphibian.dyndns.org Fri Jun 1 12:27:03 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 1 Jun 2007 13:27:03 +0100 Subject: [freenet-dev] Freenet 0.7 build 1036 Message-ID: <200706011327.11969.toad@amphibian.dyndns.org> Freenet 0.7 build 1036 is now available. The main reason for this build is a fix to auto-detection of the default language: We were setting the language code to the locale's country code, which only works in a few countries, and we weren't falling back properly to english. The build also includes some minor bugfixes, padding public keys for SSKs to a fixed size, and some internal changes. Please upgrade, and report any bugs you find! -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070601/629db3b8/attachment.pgp From toad at amphibian.dyndns.org Fri Jun 1 12:31:09 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 1 Jun 2007 13:31:09 +0100 Subject: [freenet-dev] [freenet-cvs] r13423 - trunk/freenet/src/freenet/io/comm In-Reply-To: <8b525dee0705312200y4e19a9bfmea4d242e1cdacc40@mail.gmail.com> References: <20070531212049.1DB3E47983B@emu.freenetproject.org> <8b525dee0705312200y4e19a9bfmea4d242e1cdacc40@mail.gmail.com> Message-ID: <200706011331.10458.toad@amphibian.dyndns.org> Fixed NPE. Also requests weren't working - that is also fixed now (might be the same bug). On Friday 01 June 2007 06:00, Juiceman wrote: > On 5/31/07, toad at freenetproject.org wrote: > > Author: toad > > Date: 2007-05-31 21:20:48 +0000 (Thu, 31 May 2007) > > New Revision: 13423 > > > > Modified: > > trunk/freenet/src/freenet/io/comm/MessageFilter.java > > trunk/freenet/src/freenet/io/comm/UdpSocketManager.java > > Log: > > Refactoring on MessageFilter/USM. Reduces locking a bit. Will introduce > > asynchronous MessageFilter API soon. > > This gives me a NPE > > Jun 01, 2007 04:25:01:906 (freenet.io.comm.UdpSocketManager, > UdpSocketManager packet receiver thread on port 10692, ERROR): Caught > java.lang.NullPointerException from > freenet.node.FNPPacketMangler at 75d758 > java.lang.NullPointerException > at > freenet.io.comm.UdpSocketManager.checkFilters(UdpSocketManager.java:381) at > freenet.node.FNPPacketMangler.processDecryptedData(FNPPacketMangler.java:89 >0) at freenet.node.FNPPacketMangler.tryProcess(FNPPacketMangler.java:718) at > freenet.node.FNPPacketMangler.process(FNPPacketMangler.java:118) at > freenet.io.comm.UdpSocketManager.realRun(UdpSocketManager.java:245) at > freenet.io.comm.UdpSocketManager.runLoop(UdpSocketManager.java:212) at > freenet.io.comm.UdpSocketManager.run(UdpSocketManager.java:168) -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070601/7668a880/attachment.pgp From bruthoma at student.ethz.ch Sat Jun 2 13:15:56 2007 From: bruthoma at student.ethz.ch (Thomas Bruderer) Date: Sat, 2 Jun 2007 13:15:56 +0000 (UTC) Subject: [freenet-dev] High Speed Links References: <200705212334.19325.toad@amphibian.dyndns.org> <200705221041.50024.toad@amphibian.dyndns.org> Message-ID: > On Tuesday 22 May 2007 01:15, jarvil at ... wrote: > > > Well do we really want nodes to be sending 99% of their traffic to a > > > single ubernode? That doesn't seem healthy to me. Potential attacks. > > > > Since the links of this nature would be entirely random there is no > > such thing as an "ubernode". The structure of freenet is such that > > there is no way to map topology as you have said. > > I mean on the level of an individual node. I want to mention at least something: If we have a small world topology we have many links to nearby nodes, and we have a few long links. The long links will probably have more traffic then the nearby links. If such long links would be between nodes with more capacity this would also help the rest of the network. I am aware that this would mean that the swapping algorithm would need to take bandwiths into account, and I dont suggest to do it like that, but I think it should be mentioned that not all links in the network will have the same traffic. greets Thomas / Apophis From m.rogers at cs.ucl.ac.uk Sat Jun 2 13:31:28 2007 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Sat, 02 Jun 2007 14:31:28 +0100 Subject: [freenet-dev] High Speed Links In-Reply-To: References: <200705212334.19325.toad@amphibian.dyndns.org> <200705221041.50024.toad@amphibian.dyndns.org> Message-ID: <46617130.805@cs.ucl.ac.uk> Thomas Bruderer wrote: > I want to mention at least something: If we have a small world topology we have > many links to nearby nodes, and we have a few long links. The long links will > probably have more traffic then the nearby links. If such long links would be > between nodes with more capacity this would also help the rest of the network. For what it's worth, I did some quick-and-dirty simulations a while back to test this theory - like you I thought longer links would have higher load, but it turns out the opposite is true. However my sims didn't include caching - if nearby nodes have similar items in their caches, that might relieve some of the load on short links. Cheers, Michael -------------- next part -------------- A non-text attachment was scrubbed... Name: load.png Type: image/png Size: 7896 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070602/c88997e6/attachment.png -------------- next part -------------- A non-text attachment was scrubbed... Name: Node.java Type: text/x-java Size: 1879 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070602/c88997e6/attachment.java -------------- next part -------------- A non-text attachment was scrubbed... Name: SimpleRoutingTest.java Type: text/x-java Size: 1459 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070602/c88997e6/attachment-0001.java From jarvil at gmail.com Sun Jun 3 11:52:08 2007 From: jarvil at gmail.com (jarvil at gmail.com) Date: Sun, 3 Jun 2007 21:52:08 +1000 Subject: [freenet-dev] High Speed Links In-Reply-To: <46617130.805@cs.ucl.ac.uk> References: <200705212334.19325.toad@amphibian.dyndns.org> <200705221041.50024.toad@amphibian.dyndns.org> <46617130.805@cs.ucl.ac.uk> Message-ID: I had some further though on this. When I first brought this idea up people were concerned that it might destabilise the network having a low and high speed links. On further thought this would not be the case for the following reasons. Node A has 6 links all normal speed. Node B has 6 links all normal speed. Node A and B have a seventh high speed link between them. While this would speed up there information exchange it would have no other affect on the freenet because the phenomenon is localised. The 12 normal links limit the high speed flow effects to the two nodes only and it would not propagate further down the network. The concept behind the high speed links was to enable faster propagation of data between those nodes. It would in effect be a pipeline of rapid propagation between 12 nodes. I understand this goes against the entire concept of distributed information replication via multiple sources in freenet but the propagation of data is already so slow I get the impression freenet needs it. >From what Thomas has above said it seems the smaller links are more heavily utilised. So again it seems my limited understanding of how freenet distributes data is the flaw in this request. I just cant get past the fact that 10k/s x 7-8 nodes means a long time for large data propagation. Anyway thanks for listening :-) Kind Regards Jarvil From m.rogers at cs.ucl.ac.uk Sun Jun 3 13:35:57 2007 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Sun, 03 Jun 2007 14:35:57 +0100 Subject: [freenet-dev] High Speed Links In-Reply-To: References: <200705212334.19325.toad@amphibian.dyndns.org> <200705221041.50024.toad@amphibian.dyndns.org> <46617130.805@cs.ucl.ac.uk> Message-ID: <4662C3BD.7050106@cs.ucl.ac.uk> jarvil at gmail.com wrote: > Node A has 6 links all normal speed. Node B has 6 links all normal > speed. Node A and B have a seventh high speed link between them. While > this would speed up there information exchange it would have no other > affect on the freenet because the phenomenon is localised. The 12 > normal links limit the high speed flow effects to the two nodes only > and it would not propagate further down the network. Hi Jarvil, To some extent I agree with what you're saying, we should try to make use of fast links. However we're not just trying to solve the problem of getting data from A to B as quickly as possible - we're trying to get data from the source to the destination as fast as possible, where the link between A and B is only one part of the route. Here's the problem: what if B's route to the destination is nearly as long as A's (or longer)? Then moving the data quickly from A to B doesn't achieve much, or perhaps even makes things worse. So while it is important to make use of the available bandwidth, it's not a good idea to just send data down whichever link has spare capacity - we need to find some kind of tradeoff between short routes and fast routes. There are algorithms to solve this problem if you can see the whole network (maxflow), but we need a decentralised algorithm because each node only knows about its local neighbourhood. > I just cant get > past the fact that 10k/s x 7-8 nodes means a long time for large data > propagation. Bear in mind that data isn't streamed sequentially from point to point - each block is routed independently, so all the links will be used in parallel. Cheers, Michael From bruthoma at student.ethz.ch Sun Jun 3 14:06:26 2007 From: bruthoma at student.ethz.ch (Thomas Bruderer) Date: Sun, 3 Jun 2007 14:06:26 +0000 (UTC) Subject: [freenet-dev] High Speed Links References: <200705212334.19325.toad@amphibian.dyndns.org> <200705221041.50024.toad@amphibian.dyndns.org> <46617130.805@cs.ucl.ac.uk> Message-ID: > For what it's worth, I did some quick-and-dirty simulations a while back > to test this theory - like you I thought longer links would have higher > load, but it turns out the opposite is true. Well thats nice that someone did this. However for me this simulation concerns me quite a lot. I'll probably look into the code, for me this conclusion cannot be right. --- I looked at the graph again in the first place: I think you made an interpretation error. Most messages are nearby messages, but also most links are nearby links. What I'd like to have is the number of messages per Link, and not in an absolute manner. I mean if 10 short links have 5 times more traffic, but we have only one long link, this single link has relativly still twice the traffic to a short link. But I'll look into the simulation aswell. Ty for your data. BTW: May I use your simulation-code for a base of a C# simulation? Thomas From m.rogers at cs.ucl.ac.uk Sun Jun 3 15:49:47 2007 From: m.rogers at cs.ucl.ac.uk (Michael Rogers) Date: Sun, 03 Jun 2007 16:49:47 +0100 Subject: [freenet-dev] High Speed Links In-Reply-To: References: <200705212334.19325.toad@amphibian.dyndns.org> <200705221041.50024.toad@amphibian.dyndns.org> <46617130.805@cs.ucl.ac.uk> Message-ID: <4662E31B.1030401@cs.ucl.ac.uk> Thomas Bruderer wrote: > Most messages are nearby messages, but also most links are nearby links. What > I'd like to have is the number of messages per Link, and not in an absolute > manner. Hi Thomas, The graph shows the number of messages per link, not the total for each length - I just fed the output of Node.printLoad() into gnuplot. > BTW: May I use your simulation-code for a base of a C# simulation? Please help yourself. :-) Cheers, Michael From jarvil at gmail.com Mon Jun 4 08:57:00 2007 From: jarvil at gmail.com (jarvil at gmail.com) Date: Mon, 4 Jun 2007 18:57:00 +1000 Subject: [freenet-dev] High Speed Links In-Reply-To: <4662C3BD.7050106@cs.ucl.ac.uk> References: <200705212334.19325.toad@amphibian.dyndns.org> <200705221041.50024.toad@amphibian.dyndns.org> <46617130.805@cs.ucl.ac.uk> <4662C3BD.7050106@cs.ucl.ac.uk> Message-ID: > To some extent I agree with what you're saying, we should try to make > use of fast links. However we're not just trying to solve the problem of > getting data from A to B as quickly as possible - we're trying to get > data from the source to the destination as fast as possible, where the > link between A and B is only one part of the route. Here's the problem: > what if B's route to the destination is nearly as long as A's (or > longer)? Then moving the data quickly from A to B doesn't achieve much, > or perhaps even makes things worse. So while it is important to make use > of the available bandwidth, it's not a good idea to just send data down > whichever link has spare capacity - we need to find some kind of > tradeoff between short routes and fast routes. There are algorithms to > solve this problem if you can see the whole network (maxflow), but we > need a decentralised algorithm because each node only knows about its > local neighbourhood. > Ahh I understand now! Because we are distributing in many directions at once high speed links mean nothing. If they get part 5/24 of a file distributed faster than the other 23 parts it will still take until the slowest part reaches the destination before it can be re-assembled. So you are always limited by the slowest link in the chain. In that case there is no point at all of high speed links. Thanks for clearing that up Jarvil > > I just cant get > > past the fact that 10k/s x 7-8 nodes means a long time for large data > > propagation. > > Bear in mind that data isn't streamed sequentially from point to point - > each block is routed independently, so all the links will be used in > parallel. > > Cheers, > Michael > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > From toad at amphibian.dyndns.org Wed Jun 6 18:11:27 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 6 Jun 2007 19:11:27 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes Message-ID: <200706061911.33237.toad@amphibian.dyndns.org> Recent probe data suggests a theory: Parts of the network are "rabbit holes" or "dungeons", i.e. sub-networks which are only weakly connected to the larger network. These cover a small chunk of the keyspace, say 0.36-0.41 (roughly, in the trace I had). A request for 0.5 got stuck down the rabbit hole. The gateway node at 0.436 was backed off; if the request had been able to reach that node, it would have been able to get much closer to where it should be. So the request bounced around in the dungeon, and eventually DNFed. What we need is some backtracking. At the moment backtracking only occurs when we actually run out of nodes in a pocket i.e. when it is really small. We track the best location seen so far on the request (not including dead ends i.e. RNFs), and whenever this improves we reset the HTL back to the maximum (10); otherwise it is decremented. It wasn't always this way. What we did at the beginning of 0.7 was this: Limit the number of linear hops (HTL) to 10. Only route to a node closer to the target than we are (actually I think it was only route to a node closer than the best-so-far). The problem with this approach is we'd tend to dance around at the beginning, looking for a route, wasting all our HTL. So we changed it to the above algorithm. Which does have the advantage that we won't run out of hops if we are actually getting closer to the target, and we'll have a few hops left over after we reach the best node to look around in case there have been short range network changes. The problem is, it gets stuck down rabbit holes, because there is essentially no backtracking (unless the network is really small). Both approaches are wrong. But we can have the best of both worlds: - The first time on a new node, we always route to the closest node to the target, even if it is further away from the target than the best-so-far and than this node is. Because the following hop will often be better. - When we go say 4 hops without advancing (without best-so-far improving), backtrack to the previous node with something like a DNF. - This may include the new best-so-far location if any. - After a DNF, we will try other nodes, but only if they are closer to the target than the node which is rerouting is (or the previous best-so-far if this node was the best-so-far when first entered). - A normal completion will not cause very much backtracking, because we keep the new best-so-far, and many nodes we would visit we have already visited (because of the triangles property, and this does seem to happen in practice). Having said that, there will be some forking on the way back. "Fork if closer than this node" is better than "fork if closer than the best so far" in simulations on a related project, although there are some differences. Note that dungeons are IMHO inevitable on a true darknet. Also, this should not generally increase the linear path length along which data is returned. We can probably afford the extra hops since we're not sending data down them. One worry is that inserts would send data to all the nodes; we should probably limit this. We might also want to have a total nodes visited count, with a maximum. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070606/a29dd3f7/attachment.pgp From ian at freenetproject.org Wed Jun 6 19:46:45 2007 From: ian at freenetproject.org (Ian Clarke) Date: Wed, 6 Jun 2007 14:46:45 -0500 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <200706061911.33237.toad@amphibian.dyndns.org> References: <200706061911.33237.toad@amphibian.dyndns.org> Message-ID: <823242bd0706061246g57b87ec7k9da9939c9731abb1@mail.gmail.com> Sounds promising, but you should whip up a quick simulation first to confirm that it behaves as expected. Ian. On 6/6/07, Matthew Toseland wrote: > Recent probe data suggests a theory: > > Parts of the network are "rabbit holes" or "dungeons", i.e. sub-networks which > are only weakly connected to the larger network. These cover a small chunk of > the keyspace, say 0.36-0.41 (roughly, in the trace I had). A request for 0.5 > got stuck down the rabbit hole. The gateway node at 0.436 was backed off; if > the request had been able to reach that node, it would have been able to get > much closer to where it should be. So the request bounced around in the > dungeon, and eventually DNFed. > > What we need is some backtracking. At the moment backtracking only occurs when > we actually run out of nodes in a pocket i.e. when it is really small. We > track the best location seen so far on the request (not including dead ends > i.e. RNFs), and whenever this improves we reset the HTL back to the maximum > (10); otherwise it is decremented. > > It wasn't always this way. What we did at the beginning of 0.7 was > this: Limit the number of linear hops (HTL) to 10. Only route to a node > closer to the target than we are (actually I think it was only route to a > node closer than the best-so-far). The problem with this approach is we'd > tend to dance around at the beginning, looking for a route, wasting all our > HTL. So we changed it to the above algorithm. Which does have the advantage > that we won't run out of hops if we are actually getting closer to the > target, and we'll have a few hops left over after we reach the best node to > look around in case there have been short range network changes. The problem > is, it gets stuck down rabbit holes, because there is essentially no > backtracking (unless the network is really small). > > Both approaches are wrong. But we can have the best of both worlds: > - The first time on a new node, we always route to the closest node to the > target, even if it is further away from the target than the best-so-far and > than this node is. Because the following hop will often be better. > - When we go say 4 hops without advancing (without best-so-far improving), > backtrack to the previous node with something like a DNF. > - This may include the new best-so-far location if any. > - After a DNF, we will try other nodes, but only if they are closer to the > target than the node which is rerouting is (or the previous best-so-far if > this node was the best-so-far when first entered). > - A normal completion will not cause very much backtracking, because we keep > the new best-so-far, and many nodes we would visit we have already visited > (because of the triangles property, and this does seem to happen in > practice). Having said that, there will be some forking on the way back. > > "Fork if closer than this node" is better than "fork if closer than the best > so far" in simulations on a related project, although there are some > differences. > > Note that dungeons are IMHO inevitable on a true darknet. Also, this should > not generally increase the linear path length along which data is returned. > We can probably afford the extra hops since we're not sending data down them. > One worry is that inserts would send data to all the nodes; we should > probably limit this. We might also want to have a total nodes visited count, > with a maximum. > > -- Founder and CEO, Thoof Inc Email: ian at thoof.com Office: +1 512 524 8934 x 100 Cell: +1 512 422 3588 AIM: ian.clarke at mac.com Skype: sanity From toad at amphibian.dyndns.org Wed Jun 6 20:43:52 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 6 Jun 2007 21:43:52 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <200706061911.33237.toad@amphibian.dyndns.org> References: <200706061911.33237.toad@amphibian.dyndns.org> Message-ID: <200706062144.00382.toad@amphibian.dyndns.org> After a long discussion with vivee, new proposal: On a request, we should, in addition to HTL and best-so-far, track the 3 best locations of nodes we have seen, could have visited, but haven't. When we run out of HTL, this indicates that we are either a) lost in a pocket, or b) have actually reached the end of the request. We send a DataNotFound to our previous node, just as now. But the previous node can fork a request - provided that the node it would fork to is at least as close to the target than the 3rd-best not visited node. It could be one of the nodes we didn't visit, or it could be a new option because a node came out of swapping. This should severely limit the amount of forking that happens after a normal completion, while still enabling us to backtrack our way out of rabbit holes. On Wednesday 06 June 2007 19:11, Matthew Toseland wrote: > Recent probe data suggests a theory: > > Parts of the network are "rabbit holes" or "dungeons", i.e. sub-networks > which are only weakly connected to the larger network. These cover a small > chunk of the keyspace, say 0.36-0.41 (roughly, in the trace I had). A > request for 0.5 got stuck down the rabbit hole. The gateway node at 0.436 > was backed off; if the request had been able to reach that node, it would > have been able to get much closer to where it should be. So the request > bounced around in the dungeon, and eventually DNFed. > > What we need is some backtracking. At the moment backtracking only occurs > when we actually run out of nodes in a pocket i.e. when it is really small. > We track the best location seen so far on the request (not including dead > ends i.e. RNFs), and whenever this improves we reset the HTL back to the > maximum (10); otherwise it is decremented. > > It wasn't always this way. What we did at the beginning of 0.7 was > this: Limit the number of linear hops (HTL) to 10. Only route to a node > closer to the target than we are (actually I think it was only route to a > node closer than the best-so-far). The problem with this approach is we'd > tend to dance around at the beginning, looking for a route, wasting all our > HTL. So we changed it to the above algorithm. Which does have the advantage > that we won't run out of hops if we are actually getting closer to the > target, and we'll have a few hops left over after we reach the best node to > look around in case there have been short range network changes. The > problem is, it gets stuck down rabbit holes, because there is essentially > no backtracking (unless the network is really small). > > Both approaches are wrong. But we can have the best of both worlds: > - The first time on a new node, we always route to the closest node to the > target, even if it is further away from the target than the best-so-far and > than this node is. Because the following hop will often be better. > - When we go say 4 hops without advancing (without best-so-far improving), > backtrack to the previous node with something like a DNF. > - This may include the new best-so-far location if any. > - After a DNF, we will try other nodes, but only if they are closer to the > target than the node which is rerouting is (or the previous best-so-far if > this node was the best-so-far when first entered). > - A normal completion will not cause very much backtracking, because we > keep the new best-so-far, and many nodes we would visit we have already > visited (because of the triangles property, and this does seem to happen in > practice). Having said that, there will be some forking on the way back. > > "Fork if closer than this node" is better than "fork if closer than the > best so far" in simulations on a related project, although there are some > differences. > > Note that dungeons are IMHO inevitable on a true darknet. Also, this should > not generally increase the linear path length along which data is returned. > We can probably afford the extra hops since we're not sending data down > them. One worry is that inserts would send data to all the nodes; we should > probably limit this. We might also want to have a total nodes visited > count, with a maximum. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070606/78b55086/attachment.pgp From vive at freenetproject.org Wed Jun 6 21:19:09 2007 From: vive at freenetproject.org (vive) Date: Wed, 6 Jun 2007 23:19:09 +0200 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <200706062144.00382.toad@amphibian.dyndns.org> References: <200706061911.33237.toad@amphibian.dyndns.org> <200706062144.00382.toad@amphibian.dyndns.org> Message-ID: <20070606211909.GB56932@tim.hack.org> On Wed, Jun 06, 2007 at 09:43:52PM +0100, Matthew Toseland wrote: > X-Spam-Checker-Version: SpamAssassin 3.1.7-deb (2006-10-05) on > emu.dh.bytemark.co.uk > X-Spam-Level: > X-Spam-Status: No, score=-106.9 required=5.0 tests=AWL,BAYES_00, > FORGED_RCVD_HELO,USER_IN_WHITELIST autolearn=ham version=3.1.7-deb > From: Matthew Toseland > To: Discussion of development issues > Date: Wed, 6 Jun 2007 21:43:52 +0100 > Subject: Re: [freenet-dev] Getting stuck down rabbit holes > > After a long discussion with vivee, new proposal: > On a request, we should, in addition to HTL and best-so-far, track the 3 best > locations of nodes we have seen, could have visited, but haven't. > > When we run out of HTL, this indicates that we are either a) lost in a pocket, > or b) have actually reached the end of the request. We send a DataNotFound to > our previous node, just as now. But the previous node can fork a request - > provided that the node it would fork to is at least as close to the target > than the 3rd-best not visited node. It could be one of the nodes we didn't > visit, or it could be a new option because a node came out of swapping. Part of the idea to store the 3 best positions seen-so-far but not *visited* (by the query taking greedy routing) is also to have 3 chances of backtracking and continue at a good place (instead of just backtracking and forking to a best-possible neighbor at a node as soon as possible). If the backtrack does not hit a node with a neighbor closer than the 3rd best position seen, then we backtrack until the query first reaches any(*) of the 3 nodes that had the closest neighbors-seen-so-far. From this node the query can continue to the neighbor that was not previously visited (2nd-best for greedy routing, and next step should have higher chances of leading to the destination). (*) If we take the first one encountered out of these three, then we dont have to "un-backtrack" (e.g. go down the branch again that we backtracked up from in the first case) if the query reaches *another* dead end :-) > This should severely limit the amount of forking that happens after a normal > completion, while still enabling us to backtrack our way out of rabbit holes. /vive -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 187 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070606/f231b5d6/attachment.pgp From toad at amphibian.dyndns.org Wed Jun 6 22:44:32 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Wed, 6 Jun 2007 23:44:32 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <20070606211909.GB56932@tim.hack.org> References: <200706061911.33237.toad@amphibian.dyndns.org> <200706062144.00382.toad@amphibian.dyndns.org> <20070606211909.GB56932@tim.hack.org> Message-ID: <200706062344.33468.toad@amphibian.dyndns.org> On Wednesday 06 June 2007 22:19, vive wrote: > On Wed, Jun 06, 2007 at 09:43:52PM +0100, Matthew Toseland wrote: > > X-Spam-Checker-Version: SpamAssassin 3.1.7-deb (2006-10-05) on > > emu.dh.bytemark.co.uk > > X-Spam-Level: > > X-Spam-Status: No, score=-106.9 required=5.0 tests=AWL,BAYES_00, > > FORGED_RCVD_HELO,USER_IN_WHITELIST autolearn=ham version=3.1.7-deb > > From: Matthew Toseland > > To: Discussion of development issues > > Date: Wed, 6 Jun 2007 21:43:52 +0100 > > Subject: Re: [freenet-dev] Getting stuck down rabbit holes > > > > After a long discussion with vivee, new proposal: > > On a request, we should, in addition to HTL and best-so-far, track the 3 > > best locations of nodes we have seen, could have visited, but haven't. > > > > When we run out of HTL, this indicates that we are either a) lost in a > > pocket, or b) have actually reached the end of the request. We send a > > DataNotFound to our previous node, just as now. But the previous node can > > fork a request - provided that the node it would fork to is at least as > > close to the target than the 3rd-best not visited node. It could be one > > of the nodes we didn't visit, or it could be a new option because a node > > came out of swapping. > > Part of the idea to store the 3 best positions seen-so-far but not > *visited* (by the query taking greedy routing) is also to have 3 chances of > backtracking and continue at a good place (instead of just backtracking and > forking to a best-possible neighbor at a node as soon as possible). > > If the backtrack does not hit a node with a neighbor closer than the 3rd > best position seen, then we backtrack until the query first reaches any(*) > of the 3 nodes that had the closest neighbors-seen-so-far. From this node > the query can continue to the neighbor that was not previously visited > (2nd-best for greedy routing, and next step should have higher chances of > leading to the destination). I don't see why we need this special case: We can simply fork if the fork is at least as closer to the target as the 3rd-best. This will normally result in 3 forks but it might be more or it might be less. > > (*) If we take the first one encountered out of these three, then we dont > have to "un-backtrack" (e.g. go down the branch again that we backtracked > up from in the first case) if the query reaches *another* dead end :-) > > > This should severely limit the amount of forking that happens after a > > normal completion, while still enabling us to backtrack our way out of > > rabbit holes. > > /vive -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070606/780901af/attachment.pgp From vive at freenetproject.org Thu Jun 7 08:44:24 2007 From: vive at freenetproject.org (vive) Date: Thu, 7 Jun 2007 10:44:24 +0200 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <200706062344.33468.toad@amphibian.dyndns.org> References: <200706061911.33237.toad@amphibian.dyndns.org> <200706062144.00382.toad@amphibian.dyndns.org> <20070606211909.GB56932@tim.hack.org> <200706062344.33468.toad@amphibian.dyndns.org> Message-ID: <20070607084424.GE56932@tim.hack.org> On Wed, Jun 06, 2007 at 11:44:32PM +0100, Matthew Toseland wrote: > On Wednesday 06 June 2007 22:19, vive wrote: > > On Wed, Jun 06, 2007 at 09:43:52PM +0100, Matthew Toseland wrote: > > > After a long discussion with vivee, new proposal: > > > On a request, we should, in addition to HTL and best-so-far, track the 3 > > > best locations of nodes we have seen, could have visited, but haven't. > > > > > > When we run out of HTL, this indicates that we are either a) lost in a > > > pocket, or b) have actually reached the end of the request. We send a > > > DataNotFound to our previous node, just as now. But the previous node can > > > fork a request - provided that the node it would fork to is at least as > > > close to the target than the 3rd-best not visited node. It could be one > > > of the nodes we didn't visit, or it could be a new option because a node > > > came out of swapping. > > > > Part of the idea to store the 3 best positions seen-so-far but not > > *visited* (by the query taking greedy routing) is also to have 3 chances of > > backtracking and continue at a good place (instead of just backtracking and > > forking to a best-possible neighbor at a node as soon as possible). > > > > If the backtrack does not hit a node with a neighbor closer than the 3rd > > best position seen, then we backtrack until the query first reaches any(*) > > of the 3 nodes that had the closest neighbors-seen-so-far. From this node > > the query can continue to the neighbor that was not previously visited > > (2nd-best for greedy routing, and next step should have higher chances of > > leading to the destination). > > I don't see why we need this special case: We can simply fork if the fork is > at least as closer to the target as the 3rd-best. This will normally result > in 3 forks but it might be more or it might be less. What if a node that _seemed_ to be a good option has swapped position since then? Lets say we use the rule of comparing positions against the 3rd best seen. During backtracking we will no longer see that fork as a good one, but we know it seemed so recently (perhaps it was even better than the path selected, but a node could not receive traffic due to congestion or so). That information may also be used - even if its not strictly up to date. One way to extend the rule of comparing positions would be to also attempt querying in these directions that seemed good previously. But I agree on that it's not necessary, only comparing positions may be the first way to go and see how well it works. > > (*) If we take the first one encountered out of these three, then we dont > > have to "un-backtrack" (e.g. go down the branch again that we backtracked > > up from in the first case) if the query reaches *another* dead end :-) > > > > > This should severely limit the amount of forking that happens after a > > > normal completion, while still enabling us to backtrack our way out of > > > rabbit holes. From toad at amphibian.dyndns.org Thu Jun 7 18:11:34 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Thu, 7 Jun 2007 19:11:34 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <200706061911.33237.toad@amphibian.dyndns.org> References: <200706061911.33237.toad@amphibian.dyndns.org> Message-ID: <200706071911.40072.toad@amphibian.dyndns.org> Attached is a probe trace to support the below conclusions. On Wednesday 06 June 2007 19:11, Matthew Toseland wrote: > Recent probe data suggests a theory: > > Parts of the network are "rabbit holes" or "dungeons", i.e. sub-networks > which are only weakly connected to the larger network. These cover a small > chunk of the keyspace, say 0.36-0.41 (roughly, in the trace I had). A > request for 0.5 got stuck down the rabbit hole. The gateway node at 0.436 > was backed off; if the request had been able to reach that node, it would > have been able to get much closer to where it should be. So the request > bounced around in the dungeon, and eventually DNFed. > > What we need is some backtracking. At the moment backtracking only occurs > when we actually run out of nodes in a pocket i.e. when it is really small. > We track the best location seen so far on the request (not including dead > ends i.e. RNFs), and whenever this improves we reset the HTL back to the > maximum (10); otherwise it is decremented. > > It wasn't always this way. What we did at the beginning of 0.7 was > this: Limit the number of linear hops (HTL) to 10. Only route to a node > closer to the target than we are (actually I think it was only route to a > node closer than the best-so-far). The problem with this approach is we'd > tend to dance around at the beginning, looking for a route, wasting all our > HTL. So we changed it to the above algorithm. Which does have the advantage > that we won't run out of hops if we are actually getting closer to the > target, and we'll have a few hops left over after we reach the best node to > look around in case there have been short range network changes. The > problem is, it gets stuck down rabbit holes, because there is essentially > no backtracking (unless the network is really small). > > Both approaches are wrong. But we can have the best of both worlds: > - The first time on a new node, we always route to the closest node to the > target, even if it is further away from the target than the best-so-far and > than this node is. Because the following hop will often be better. > - When we go say 4 hops without advancing (without best-so-far improving), > backtrack to the previous node with something like a DNF. > - This may include the new best-so-far location if any. > - After a DNF, we will try other nodes, but only if they are closer to the > target than the node which is rerouting is (or the previous best-so-far if > this node was the best-so-far when first entered). > - A normal completion will not cause very much backtracking, because we > keep the new best-so-far, and many nodes we would visit we have already > visited (because of the triangles property, and this does seem to happen in > practice). Having said that, there will be some forking on the way back. > > "Fork if closer than this node" is better than "fork if closer than the > best so far" in simulations on a related project, although there are some > differences. > > Note that dungeons are IMHO inevitable on a true darknet. Also, this should > not generally increase the linear path length along which data is returned. > We can probably afford the extra hops since we're not sending data down > them. One worry is that inserts would send data to all the nodes; we should > probably limit this. We might also want to have a total nodes visited > count, with a maximum. -------------- next part -------------- A non-text attachment was scrubbed... Name: trace.txt.zip Type: application/x-zip Size: 2442 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070607/5cd21153/attachment.bin -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070607/5cd21153/attachment.pgp From jargonautti at hotmail.com Thu Jun 7 20:23:48 2007 From: jargonautti at hotmail.com (Jusa Saari) Date: Thu, 07 Jun 2007 23:23:48 +0300 Subject: [freenet-dev] Getting stuck down rabbit holes References: <200706061911.33237.toad@amphibian.dyndns.org> Message-ID: On Wed, 06 Jun 2007 19:11:27 +0100, Matthew Toseland wrote: > Recent probe data suggests a theory: > > Parts of the network are "rabbit holes" or "dungeons", i.e. sub-networks > which are only weakly connected to the larger network. These cover a small > chunk of the keyspace, say 0.36-0.41 (roughly, in the trace I had). A > request for 0.5 got stuck down the rabbit hole. The gateway node at 0.436 > was backed off; if the request had been able to reach that node, it would > have been able to get much closer to where it should be. So the request > bounced around in the dungeon, and eventually DNFed. > > What we need is some backtracking. At the moment backtracking only occurs > when we actually run out of nodes in a pocket i.e. when it is really > small. We track the best location seen so far on the request (not > including dead ends i.e. RNFs), and whenever this improves we reset the > HTL back to the maximum (10); otherwise it is decremented. No, what you need is an opennet. Having a "sparse" network with plenty of "leaves" only connected to the rest of a network through a single node is a natural feature of a darknet. The leaves are made of people who happened to be in #freenet-refs at the same time, exchanged refs, and left; the connecting nodes are those who are running automated ref exchange scripts and therefore get connected to the new leaves as they are formed. Simply backing away from the leaves is going to overload the single connecting node since all the traffic to and from the leaves is going to go through it; and of course if that node happens to go offline for any reason the network will splinter. A darknet is never going to work well; the network topology forces the majority of request through a small amount of well-connected nodes. They are going to get overloaded, and even if they won't, the network will be splintered if they are taken offline, which is easy since they have to be semi-public to get well connected in the first place. This makes the "dark" Freenet easy to take down. So, the darknet is fragile, won't ever perform well, and is a pain in the ass to get into and stay in. No amount of tweaking can get over these fundamental inherent flaws in a darknet approach. Only an enabled-by-default opennet can solve them. Oh well, it's not my problem, since I'm not running Freenet anymore; the hassle of having to get new refs from #freenet-refs every few days as the old ones go offline was simply too much. I guess I'll be checking back every couple of months to see if the sanity of openness has returned or if the Freenet is still in the dark. > It wasn't always this way. What we did at the beginning of 0.7 was this: > Limit the number of linear hops (HTL) to 10. Only route to a node closer > to the target than we are (actually I think it was only route to a node > closer than the best-so-far). The problem with this approach is we'd tend > to dance around at the beginning, looking for a route, wasting all our > HTL. So we changed it to the above algorithm. Which does have the > advantage that we won't run out of hops if we are actually getting closer > to the target, and we'll have a few hops left over after we reach the best > node to look around in case there have been short range network changes. > The problem is, it gets stuck down rabbit holes, because there is > essentially no backtracking (unless the network is really small). > > Both approaches are wrong. But we can have the best of both worlds: - The > first time on a new node, we always route to the closest node to the > target, even if it is further away from the target than the best-so-far > and than this node is. Because the following hop will often be better. - > When we go say 4 hops without advancing (without best-so-far improving), > backtrack to the previous node with something like a DNF. - This may > include the new best-so-far location if any. - After a DNF, we will try > other nodes, but only if they are closer to the target than the node which > is rerouting is (or the previous best-so-far if this node was the > best-so-far when first entered). - A normal completion will not cause very > much backtracking, because we keep the new best-so-far, and many nodes we > would visit we have already visited (because of the triangles property, > and this does seem to happen in practice). Having said that, there will be > some forking on the way back. > > "Fork if closer than this node" is better than "fork if closer than the > best so far" in simulations on a related project, although there are some > differences. > > Note that dungeons are IMHO inevitable on a true darknet. Also, this > should not generally increase the linear path length along which data is > returned. We can probably afford the extra hops since we're not sending > data down them. One worry is that inserts would send data to all the > nodes; we should probably limit this. We might also want to have a total > nodes visited count, with a maximum._______________________________________________ Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From nextgens at freenetproject.org Thu Jun 7 20:44:44 2007 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Thu, 7 Jun 2007 22:44:44 +0200 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: References: <200706061911.33237.toad@amphibian.dyndns.org> Message-ID: <20070607204443.GA5511@freenetproject.org> * Jusa Saari [2007-06-07 23:23:48]: > On Wed, 06 Jun 2007 19:11:27 +0100, Matthew Toseland wrote: > > > Recent probe data suggests a theory: > > > > Parts of the network are "rabbit holes" or "dungeons", i.e. sub-networks > > which are only weakly connected to the larger network. These cover a small > > chunk of the keyspace, say 0.36-0.41 (roughly, in the trace I had). A > > request for 0.5 got stuck down the rabbit hole. The gateway node at 0.436 > > was backed off; if the request had been able to reach that node, it would > > have been able to get much closer to where it should be. So the request > > bounced around in the dungeon, and eventually DNFed. > > > > What we need is some backtracking. At the moment backtracking only occurs > > when we actually run out of nodes in a pocket i.e. when it is really > > small. We track the best location seen so far on the request (not > > including dead ends i.e. RNFs), and whenever this improves we reset the > > HTL back to the maximum (10); otherwise it is decremented. > > No, what you need is an opennet. Having a "sparse" network with plenty of > "leaves" only connected to the rest of a network through a single node is > a natural feature of a darknet. The leaves are made of people who > happened to be in #freenet-refs at the same time, exchanged refs, and > left; the connecting nodes are those who are running automated ref > exchange scripts and therefore get connected to the new leaves as they > are formed. Simply backing away from the leaves is going to overload the > single connecting node since all the traffic to and from the leaves is > going to go through it; and of course if that node happens to go offline > for any reason the network will splinter. > > A darknet is never going to work well; the network topology forces the > majority of request through a small amount of well-connected nodes. They > are going to get overloaded, and even if they won't, the network will be > splintered if they are taken offline, which is easy since they have to be > semi-public to get well connected in the first place. This makes the > "dark" Freenet easy to take down. > > So, the darknet is fragile, won't ever perform well, and is a pain in the > ass to get into and stay in. No amount of tweaking can get over these > fundamental inherent flaws in a darknet approach. Only an > enabled-by-default opennet can solve them. > > Oh well, it's not my problem, since I'm not running Freenet anymore; the > hassle of having to get new refs from #freenet-refs every few days as the > old ones go offline was simply too much. I guess I'll be checking back > every couple of months to see if the sanity of openness has returned or if > the Freenet is still in the dark. > You miss the point here. No one has ever argued that a darknet built like the current one can work... The routing algorithm is based on small worlds properties, if the darknet isn't a small world it won't work; full stop. The darknet approach could work if people were really playing the game and building it like a social network because such networks are small worlds... The problem is that the network has to reach a critical mass beforehand so that connecting to real life friends becomes possible. Implementing a workaround (opennet, backtracking, ...) is only a way of fixing temporarily the topology to the expense of both liberty (it has to be the default behaviour as you pointed out) and safety (everyone knows that the opennet approach has design caveats). "They that can give up essential liberty to obtain a little temporary safety deserve neither liberty nor safety" -- Benjamin Franklin NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070607/59930800/attachment.pgp From ian at locut.us Thu Jun 7 21:34:31 2007 From: ian at locut.us (Ian Clarke) Date: Thu, 7 Jun 2007 16:34:31 -0500 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <20070607204443.GA5511@freenetproject.org> References: <200706061911.33237.toad@amphibian.dyndns.org> <20070607204443.GA5511@freenetproject.org> Message-ID: <823242bd0706071434u191a7901g55ef6d5295024591@mail.gmail.com> On 6/7/07, Florent Daigni?re wrote: > * Jusa Saari [2007-06-07 23:23:48]: > Implementing a workaround (opennet, backtracking, ...) is only a way of > fixing temporarily the topology to the expense of both liberty (it has > to be the default behaviour as you pointed out) and safety (everyone > knows that the opennet approach has design caveats). > > "They that can give up essential liberty to obtain a little temporary > safety deserve neither liberty nor safety" > -- Benjamin Franklin Your use of the Franklin quote suggests that you are looking at this backwards. The alternative here is not between users using an opennet or a darknet, its between them using an opennet, or another solution that is far worse (such as a public proxy). This is the simple reality of the situation that we see time and time again whenever we bother to listen to our users. Consequently, by giving people the option of an opennet, we aren't inviting them to give up liberty, we are inviting them to increase it. Most sane people think teenagers should have access to condoms, but some people think its a bad idea, claiming that giving teenagers access to condoms will encourage them to have sex. The point, obviously, is that teenagers will have sex anyway, the only question is whether it will be safe sex. Substitute opennet for condoms, and darknet for abstinence, and you see that many of those arguing against opennet are following the exact same wrongheaded line of reasoning as those that disagree with allowing teenagers access to condoms. Ian. -- Founder and CEO, Thoof Inc Email: ian at thoof.com Office: +1 512 524 8934 x 100 Cell: +1 512 422 3588 AIM: ian.clarke at mac.com Skype: sanity From vive at freenetproject.org Thu Jun 7 21:42:38 2007 From: vive at freenetproject.org (vive) Date: Thu, 7 Jun 2007 23:42:38 +0200 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: References: <200706061911.33237.toad@amphibian.dyndns.org> Message-ID: <20070607214238.GA76797@tim.hack.org> On Thu, Jun 07, 2007 at 11:23:48PM +0300, Jusa Saari wrote: > On Wed, 06 Jun 2007 19:11:27 +0100, Matthew Toseland wrote: > > > Recent probe data suggests a theory: > > > > Parts of the network are "rabbit holes" or "dungeons", i.e. sub-networks > > which are only weakly connected to the larger network. These cover a small > > chunk of the keyspace, say 0.36-0.41 (roughly, in the trace I had). A > > request for 0.5 got stuck down the rabbit hole. The gateway node at 0.436 > > was backed off; if the request had been able to reach that node, it would > > have been able to get much closer to where it should be. So the request > > bounced around in the dungeon, and eventually DNFed. > > > > What we need is some backtracking. At the moment backtracking only occurs > > when we actually run out of nodes in a pocket i.e. when it is really > > small. We track the best location seen so far on the request (not > > including dead ends i.e. RNFs), and whenever this improves we reset the > > HTL back to the maximum (10); otherwise it is decremented. > > No, what you need is an opennet. Having a "sparse" network with plenty of > "leaves" only connected to the rest of a network through a single node is > a natural feature of a darknet. The leaves are made of people who Hi, the current growth model is not the only possible under a darknet approach. Yes, a problem is how the network grows. If no similar topology is in place from the start there is _no way_ the network (when scaling it) can become efficiently navigable by swapping positions trying to fit an ideal model of topology which does not exist (the Kleinberg model where it works well). One way that IMHO would grow the network better (assuming one wants to stay in the darknet approach) would be connecting growth to a (an existing?) 'killer app' that lets you easily invite your friends into the network. Currently the barrier is too high, and when wanting to get into the network you have no real incentive of getting friends connected compared to going to #freenet-refs I have heard of at least one proposal to make the network growth easier for friend-to-friend structures; there was some discussion on a jabber- plugin for easily letting you invite your friends. But that would only be a start, a nice one to evaluate though. > happened to be in #freenet-refs at the same time, exchanged refs, and > left; the connecting nodes are those who are running automated ref > exchange scripts and therefore get connected to the new leaves as they > are formed. Simply backing away from the leaves is going to overload the > single connecting node since all the traffic to and from the leaves is > going to go through it; and of course if that node happens to go offline > for any reason the network will splinter. Agreed, the load/congestion problem and the topology problems seem to be very related. Not just that under a poor topology much load may be placed on few 'good' nodes, but that we cannot expect the greedy routing to work very good at all as in the Kleinberg model (=> longer chains => more data for network). cheers, /v From toad at amphibian.dyndns.org Thu Jun 7 23:06:42 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 8 Jun 2007 00:06:42 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <823242bd0706071434u191a7901g55ef6d5295024591@mail.gmail.com> References: <200706061911.33237.toad@amphibian.dyndns.org> <20070607204443.GA5511@freenetproject.org> <823242bd0706071434u191a7901g55ef6d5295024591@mail.gmail.com> Message-ID: <200706080006.43381.toad@amphibian.dyndns.org> Nonetheless the general thrust of Jusa's argument was that there is no point having darknet. He's wrong. Opennet is a means to an end, not the be all and end all. On Thursday 07 June 2007 22:34, Ian Clarke wrote: > On 6/7/07, Florent Daigni?re wrote: > > * Jusa Saari [2007-06-07 23:23:48]: > > Implementing a workaround (opennet, backtracking, ...) is only a way of > > fixing temporarily the topology to the expense of both liberty (it has > > to be the default behaviour as you pointed out) and safety (everyone > > knows that the opennet approach has design caveats). > > > > "They that can give up essential liberty to obtain a little temporary > > safety deserve neither liberty nor safety" > > -- Benjamin Franklin > > Your use of the Franklin quote suggests that you are looking at this > backwards. > > The alternative here is not between users using an opennet or a > darknet, its between them using an opennet, or another solution that > is far worse (such as a public proxy). This is the simple reality of > the situation that we see time and time again whenever we bother to > listen to our users. > > Consequently, by giving people the option of an opennet, we aren't > inviting them to give up liberty, we are inviting them to increase it. > > Most sane people think teenagers should have access to condoms, but > some people think its a bad idea, claiming that giving teenagers > access to condoms will encourage them to have sex. The point, > obviously, is that teenagers will have sex anyway, the only question > is whether it will be safe sex. > > Substitute opennet for condoms, and darknet for abstinence, and you > see that many of those arguing against opennet are following the exact > same wrongheaded line of reasoning as those that disagree with > allowing teenagers access to condoms. > > Ian. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070608/389709ee/attachment.pgp From toad at amphibian.dyndns.org Thu Jun 7 23:05:41 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 8 Jun 2007 00:05:41 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: References: <200706061911.33237.toad@amphibian.dyndns.org> Message-ID: <200706080005.46991.toad@amphibian.dyndns.org> On Thursday 07 June 2007 21:23, Jusa Saari wrote: > On Wed, 06 Jun 2007 19:11:27 +0100, Matthew Toseland wrote: > > Recent probe data suggests a theory: > > > > Parts of the network are "rabbit holes" or "dungeons", i.e. sub-networks > > which are only weakly connected to the larger network. These cover a > > small chunk of the keyspace, say 0.36-0.41 (roughly, in the trace I had). > > A request for 0.5 got stuck down the rabbit hole. The gateway node at > > 0.436 was backed off; if the request had been able to reach that node, it > > would have been able to get much closer to where it should be. So the > > request bounced around in the dungeon, and eventually DNFed. > > > > What we need is some backtracking. At the moment backtracking only occurs > > when we actually run out of nodes in a pocket i.e. when it is really > > small. We track the best location seen so far on the request (not > > including dead ends i.e. RNFs), and whenever this improves we reset the > > HTL back to the maximum (10); otherwise it is decremented. > > No, what you need is an opennet. Having a "sparse" network with plenty of > "leaves" only connected to the rest of a network through a single node is > a natural feature of a darknet. The leaves are made of people who > happened to be in #freenet-refs at the same time, exchanged refs, and > left; the connecting nodes are those who are running automated ref > exchange scripts and therefore get connected to the new leaves as they > are formed. Simply backing away from the leaves is going to overload the > single connecting node since all the traffic to and from the leaves is > going to go through it; and of course if that node happens to go offline > for any reason the network will splinter. > > A darknet is never going to work well; the network topology forces the > majority of request through a small amount of well-connected nodes. They > are going to get overloaded, and even if they won't, the network will be > splintered if they are taken offline, which is easy since they have to be > semi-public to get well connected in the first place. This makes the > "dark" Freenet easy to take down. We are not talking about darknet vs opennet. If you want an opennet go use Tor hidden services, or I2P, or any of the numerous opennet p2p's out there. In the long term, if there is no darknet, THERE IS NO FREENET. It doesn't matter if we have an opennet that outperforms BitTorrent over Tor (which incidentally isn't that hard!): It will be blocked using the systems that are already in place in most western countries, let alone China et al. Now, I might concede that many of the problems of the present network are caused by the pseudo-opennet growth of the present "darknet". But we have excellent reasons to believe that if it were in fact a true darknet, it would be a small world network and therefore navigable. That's not to say that we shouldn't implement opennet. Opennet is a great way to get people onto the network. But it is not a long term solution. And it will very likely suffer from all sorts of serious problems, just as 0.5 did. Backtracking will help on a true darknet, because it is likely to be an imperfect network, with sub-networks constantly joining up. It will help on a hybrid network, because not all of the darknet sub-networks will be strongly connected to the opennet. And it will probably help even on a pure opennet, because real world stresses mean that the topology on a pure opennet will not be perfect. It is much easier to debug many of these routing and load management problems on a darknet, even a pseudo-opennet darknet, because we have much slower connection churn than on an opennet. Once opennet is implemented, while some problems will be reduced, many other problems will spring up in their place. Problems which we never managed to adequately solve on the 0.5 opennet. For example, you can get onto (and up to speed on) the 0.7 pseudo-darknet a good deal faster than onto the 0.5 opennet. But we will implement opennet - after we have at least tried to sort out some of the current problems, and demonstrated experimentally and observationally that the best way forward is opennet, that we've fixed all the easy stuff. At least, that's the plan as I understand it. > > So, the darknet is fragile, won't ever perform well, and is a pain in the > ass to get into and stay in. No amount of tweaking can get over these > fundamental inherent flaws in a darknet approach. Only an > enabled-by-default opennet can solve them. Opennet will never be enabled by default. We will ask the user whether they want opennet. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070608/bd4c22a9/attachment.pgp From ian at locut.us Thu Jun 7 23:20:04 2007 From: ian at locut.us (Ian Clarke) Date: Thu, 7 Jun 2007 18:20:04 -0500 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <200706080005.46991.toad@amphibian.dyndns.org> References: <200706061911.33237.toad@amphibian.dyndns.org> <200706080005.46991.toad@amphibian.dyndns.org> Message-ID: <823242bd0706071620m2b056ee7wc31b6de450f7bdd4@mail.gmail.com> On 6/7/07, Matthew Toseland wrote: > It doesn't matter > if we have an opennet that outperforms BitTorrent over Tor On that subject, NEVER try using BitTorrent over Tor unless for some reason you hate Tor and are trying to do a denial of service attack on it. Ian. -- Founder and CEO, Thoof Inc Email: ian at thoof.com Office: +1 512 524 8934 x 100 Cell: +1 512 422 3588 AIM: ian.clarke at mac.com Skype: sanity From toad at amphibian.dyndns.org Thu Jun 7 23:55:17 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 8 Jun 2007 00:55:17 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <823242bd0706071620m2b056ee7wc31b6de450f7bdd4@mail.gmail.com> References: <200706061911.33237.toad@amphibian.dyndns.org> <200706080005.46991.toad@amphibian.dyndns.org> <823242bd0706071620m2b056ee7wc31b6de450f7bdd4@mail.gmail.com> Message-ID: <200706080055.28919.toad@amphibian.dyndns.org> On Friday 08 June 2007 00:20, Ian Clarke wrote: > On 6/7/07, Matthew Toseland wrote: > > It doesn't matter > > if we have an opennet that outperforms BitTorrent over Tor > > On that subject, NEVER try using BitTorrent over Tor unless for some > reason you hate Tor and are trying to do a denial of service attack on > it. :) Seriously, some of the elements in our current routing system e.g. the HTL / lack of backoff haven't really been tested properly or even simulated properly; they're based on a series of assumptions. Hopefully we can rectify that, but an easy option at this point is to just try out a different algorithm and see what effect it has on probe requests. Once we have update over mandatory it should be relatively easy to do such things. > > Ian. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070608/a0835fd2/attachment.pgp From jargonautti at hotmail.com Fri Jun 8 12:11:02 2007 From: jargonautti at hotmail.com (Jusa Saari) Date: Fri, 08 Jun 2007 15:11:02 +0300 Subject: [freenet-dev] Getting stuck down rabbit holes References: <200706061911.33237.toad@amphibian.dyndns.org> <200706080005.46991.toad@amphibian.dyndns.org> Message-ID: On Fri, 08 Jun 2007 00:05:41 +0100, Matthew Toseland wrote: > On Thursday 07 June 2007 21:23, Jusa Saari wrote: >> On Wed, 06 Jun 2007 19:11:27 +0100, Matthew Toseland wrote: >> > Recent probe data suggests a theory: >> > >> > Parts of the network are "rabbit holes" or "dungeons", i.e. >> > sub-networks which are only weakly connected to the larger network. >> > These cover a small chunk of the keyspace, say 0.36-0.41 (roughly, in >> > the trace I had). A request for 0.5 got stuck down the rabbit hole. >> > The gateway node at 0.436 was backed off; if the request had been able >> > to reach that node, it would have been able to get much closer to >> > where it should be. So the request bounced around in the dungeon, and >> > eventually DNFed. >> > >> > What we need is some backtracking. At the moment backtracking only >> > occurs when we actually run out of nodes in a pocket i.e. when it is >> > really small. We track the best location seen so far on the request >> > (not including dead ends i.e. RNFs), and whenever this improves we >> > reset the HTL back to the maximum (10); otherwise it is decremented. >> >> No, what you need is an opennet. Having a "sparse" network with plenty >> of "leaves" only connected to the rest of a network through a single >> node is a natural feature of a darknet. The leaves are made of people >> who happened to be in #freenet-refs at the same time, exchanged refs, >> and left; the connecting nodes are those who are running automated ref >> exchange scripts and therefore get connected to the new leaves as they >> are formed. Simply backing away from the leaves is going to overload the >> single connecting node since all the traffic to and from the leaves is >> going to go through it; and of course if that node happens to go offline >> for any reason the network will splinter. >> >> A darknet is never going to work well; the network topology forces the >> majority of request through a small amount of well-connected nodes. They >> are going to get overloaded, and even if they won't, the network will be >> splintered if they are taken offline, which is easy since they have to >> be semi-public to get well connected in the first place. This makes the >> "dark" Freenet easy to take down. > > We are not talking about darknet vs opennet. If you want an opennet go use > Tor hidden services, or I2P, or any of the numerous opennet p2p's out > there. In the long term, if there is no darknet, THERE IS NO FREENET. It > doesn't matter if we have an opennet that outperforms BitTorrent over Tor > (which incidentally isn't that hard!): It will be blocked using the > systems that are already in place in most western countries, let alone > China et al. Wihtout opennet, there won't be Freenet either. The darknet is too much trouble to get into and stay in. And a repressive regime can always simply look at machines which are exchanging encrypted traffick with nonstandard ports (non-https) in other machines. > Now, I might concede that many of the problems of the present network are > caused by the pseudo-opennet growth of the present "darknet". But we have > excellent reasons to believe that if it were in fact a true darknet, it > would be a small world network and therefore navigable. No. The pseudo-opennet growth is what enables there to be a network at all. Freenet lacks sufficient user base that anyone wanting to use it would be likely to find any real-life friends already using it; and even if there were sufficient numbers, going through your friends asking "do you run Freenet" would simply be too much hassle, _especially_ since it might risk them thinking that you're wanting to look at naughty stuff. Besides, there can never be a true darknet, for the simple reason that in order to connect to _anyone_ will require telling them you're running a node. The only question is the exact shade of gray a particular network will be; currently, Freenet users are stumbling in the dark :(. > That's not to say that we shouldn't implement opennet. Opennet is a great > way to get people onto the network. But it is not a long term solution. It is the only solution that has a snowballs chance in Hell of actually working. > And it will very likely suffer from all sorts of serious problems, just as > 0.5 did. Backtracking will help on a true darknet, because it is likely to > be an imperfect network, with sub-networks constantly joining up. It will > help on a hybrid network, because not all of the darknet sub-networks will > be strongly connected to the opennet. And it will probably help even on a > pure opennet, because real world stresses mean that the topology on a pure > opennet will not be perfect. > > It is much easier to debug many of these routing and load management > problems on a darknet, even a pseudo-opennet darknet, because we have much > slower connection churn than on an opennet. Once opennet is implemented, > while some problems will be reduced, many other problems will spring up in > their place. Problems which we never managed to adequately solve on the > 0.5 opennet. For example, you can get onto (and up to speed on) the 0.7 > pseudo-darknet a good deal faster than onto the 0.5 opennet. But we will Assuming you know some people who run Freenet and make this fact semi-public. Otherwise it's lurking in #freenet-refs for you. > implement opennet - after we have at least tried to sort out some of the > current problems, and demonstrated experimentally and observationally that > the best way forward is opennet, that we've fixed all the easy stuff. > > At least, that's the plan as I understand it. >> >> So, the darknet is fragile, won't ever perform well, and is a pain in >> the ass to get into and stay in. No amount of tweaking can get over >> these fundamental inherent flaws in a darknet approach. Only an >> enabled-by-default opennet can solve them. > > Opennet will never be enabled by default. We will ask the user whether > they want opennet._______________________________________________ Devl mailing list Then I guess Freenet will never grow large enough to accomplish much of anything. > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From jargonautti at hotmail.com Fri Jun 8 12:17:49 2007 From: jargonautti at hotmail.com (Jusa Saari) Date: Fri, 08 Jun 2007 15:17:49 +0300 Subject: [freenet-dev] Getting stuck down rabbit holes References: <200706061911.33237.toad@amphibian.dyndns.org> <20070607204443.GA5511@freenetproject.org> <823242bd0706071434u191a7901g55ef6d5295024591@mail.gmail.com> <200706080006.43381.toad@amphibian.dyndns.org> Message-ID: On Fri, 08 Jun 2007 00:06:42 +0100, Matthew Toseland wrote: > Nonetheless the general thrust of Jusa's argument was that there is no > point having darknet. He's wrong. Opennet is a means to an end, not the be > all and end all. More to the point, I'm trying to argue that having a global darknet is not an attainable goal, and pushing for it will accomplish nothing except hinder Freenet growth and functionality. The chances for anyone wanting to join the network to have IRL friends already running it are nearly nonexistent due to the small amounts of such users currently in the network; and remember, you need at least three such contacts before your node is actually routing. And of course opennet is a means to an end. So is the whole Freenet, and every other computer program in existence, as well as the computer itself. > On Thursday 07 June 2007 22:34, Ian Clarke wrote: >> On 6/7/07, Florent Daigni?re >> wrote: >> > * Jusa Saari >> > [2007-06-07 23:23:48]: Implementing a workaround (opennet, >> > backtracking, ...) is only a way of fixing temporarily the topology to >> > the expense of both liberty (it has to be the default behaviour as you >> > pointed out) and safety (everyone knows that the opennet approach has >> > design caveats). >> > >> > "They that can give up essential liberty to obtain a little temporary >> > safety deserve neither liberty nor safety" -- Benjamin Franklin >> >> Your use of the Franklin quote suggests that you are looking at this >> backwards. >> >> The alternative here is not between users using an opennet or a darknet, >> its between them using an opennet, or another solution that is far worse >> (such as a public proxy). This is the simple reality of the situation >> that we see time and time again whenever we bother to listen to our >> users. >> >> Consequently, by giving people the option of an opennet, we aren't >> inviting them to give up liberty, we are inviting them to increase it. >> >> Most sane people think teenagers should have access to condoms, but some >> people think its a bad idea, claiming that giving teenagers access to >> condoms will encourage them to have sex. The point, obviously, is that >> teenagers will have sex anyway, the only question is whether it will be >> safe sex. >> >> Substitute opennet for condoms, and darknet for abstinence, and you see >> that many of those arguing against opennet are following the exact same >> wrongheaded line of reasoning as those that disagree with allowing >> teenagers access to condoms. >> >> Ian._______________________________________________ Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From Colin at sq7.org Fri Jun 8 12:25:37 2007 From: Colin at sq7.org (Colin Davis) Date: Fri, 08 Jun 2007 08:25:37 -0400 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: References: <200706061911.33237.toad@amphibian.dyndns.org> <20070607204443.GA5511@freenetproject.org> <823242bd0706071434u191a7901g55ef6d5295024591@mail.gmail.com> <200706080006.43381.toad@amphibian.dyndns.org> Message-ID: <46694AC1.8060406@sq7.org> Jusa- I don't think this line of argument is productive. ;) They've had this argument 10,000 times, and I don't think Matthew's position is going to change. He fundamentally believes that ANY opennet system could be easily blocked if the situation ever came to it. He has a good point, even if you don't agree with him. It's much easier to block an opennet. Your point is that without a critical mass, Freenet is of limited usefulness- It's hard to FIND friends who use Freenet, which makes a global darknet difficult. This is also true. I don't know that having this same discussion over and over will being up and new data, or sway anyone ;) I look at it, and know that opennet is necessary for the darknet to prosper.. Once Opennet is in place, people have a larger incentive to run Freenet. This means that they add permanent nodes, which is good. This build the larger base of Freenet users, which we need to make a darknet work. Two opennet users can talk to one another offline, and agree to exchange a Darknet Link. This grows the number of links that aren't posted about anywhere. Later on, as new users join, they're friends already ARE using freenet. This means that they can exchange darknet links with people they know, without ever having to run opennet. Opennet is the key to getting a the installed base which allows darknets to work. -Colin Jusa Saari wrote: > On Fri, 08 Jun 2007 00:06:42 +0100, Matthew Toseland wrote: > >> Nonetheless the general thrust of Jusa's argument was that there is no >> point having darknet. He's wrong. Opennet is a means to an end, not the be >> all and end all. > > More to the point, I'm trying to argue that having a global darknet is not > an attainable goal, and pushing for it will accomplish nothing except > hinder Freenet growth and functionality. The chances for anyone wanting to > join the network to have IRL friends already running it are nearly > nonexistent due to the small amounts of such users currently in the > network; and remember, you need at least three such contacts before your > node is actually routing. > > And of course opennet is a means to an end. So is the whole Freenet, and > every other computer program in existence, as well as the computer itself. > >> On Thursday 07 June 2007 22:34, Ian Clarke wrote: >>> On 6/7/07, Florent Daigni?re >>> wrote: >>>> * Jusa Saari >>>> [2007-06-07 23:23:48]: Implementing a workaround (opennet, >>>> backtracking, ...) is only a way of fixing temporarily the topology to >>>> the expense of both liberty (it has to be the default behaviour as you >>>> pointed out) and safety (everyone knows that the opennet approach has >>>> design caveats). >>>> >>>> "They that can give up essential liberty to obtain a little temporary >>>> safety deserve neither liberty nor safety" -- Benjamin Franklin >>> Your use of the Franklin quote suggests that you are looking at this >>> backwards. >>> >>> The alternative here is not between users using an opennet or a darknet, >>> its between them using an opennet, or another solution that is far worse >>> (such as a public proxy). This is the simple reality of the situation >>> that we see time and time again whenever we bother to listen to our >>> users. >>> >>> Consequently, by giving people the option of an opennet, we aren't >>> inviting them to give up liberty, we are inviting them to increase it. >>> >>> Most sane people think teenagers should have access to condoms, but some >>> people think its a bad idea, claiming that giving teenagers access to >>> condoms will encourage them to have sex. The point, obviously, is that >>> teenagers will have sex anyway, the only question is whether it will be >>> safe sex. >>> >>> Substitute opennet for condoms, and darknet for abstinence, and you see >>> that many of those arguing against opennet are following the exact same >>> wrongheaded line of reasoning as those that disagree with allowing >>> teenagers access to condoms. >>> >>> Ian._______________________________________________ Devl mailing list >> Devl at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From nextgens at freenetproject.org Fri Jun 8 13:05:57 2007 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Fri, 8 Jun 2007 15:05:57 +0200 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <46694AC1.8060406@sq7.org> References: <200706061911.33237.toad@amphibian.dyndns.org> <20070607204443.GA5511@freenetproject.org> <823242bd0706071434u191a7901g55ef6d5295024591@mail.gmail.com> <200706080006.43381.toad@amphibian.dyndns.org> <46694AC1.8060406@sq7.org> Message-ID: <20070608130557.GB5559@freenetproject.org> * Colin Davis [2007-06-08 08:25:37]: [snip.] > I look at it, and know that opennet is necessary for the darknet to > prosper.. Once Opennet is in place, people have a larger incentive to > run Freenet. This means that they add permanent nodes, which is good. Opennet will add churn to the equation and will make most of the network harvestable, that's for sure; creating and linking more permanent nodes is a hope more than anything else. > > This build the larger base of Freenet users, which we need to make a > darknet work. Two opennet users can talk to one another offline, and > agree to exchange a Darknet Link. This grows the number of links that > aren't posted about anywhere. That won't *ever* happen. Why ? because the whole point of freenet is preventing anyone to link a "virtual over freeenet" identity to a physical beeing... And for a darknet link to be established you have to exchange informations about your node which can be linked to your real identity => you won't ever use freenet to establish new darknet links unless you don't worship your anonymity > > Later on, as new users join, they're friends already ARE using freenet. > This means that they can exchange darknet links with people they know, > without ever having to run opennet. > If we don't tell/teach users about the risks of connecting to strangers, they won't connect to their friends : they won't have any incentive to do so (opennet would be the default, easiest way of getting bootstraped). > > Opennet is the key to getting a the installed base which allows darknets > to work. > I won't comment once again on that one. NextGen$ -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070608/350cbfce/attachment.pgp From Colin at sq7.org Fri Jun 8 13:25:15 2007 From: Colin at sq7.org (Colin Davis) Date: Fri, 08 Jun 2007 09:25:15 -0400 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <20070608130557.GB5559@freenetproject.org> References: <200706061911.33237.toad@amphibian.dyndns.org> <20070607204443.GA5511@freenetproject.org> <823242bd0706071434u191a7901g55ef6d5295024591@mail.gmail.com> <200706080006.43381.toad@amphibian.dyndns.org> <46694AC1.8060406@sq7.org> <20070608130557.GB5559@freenetproject.org> Message-ID: <466958BB.9010001@sq7.org> We disagree on user behavior and motivation, and I accept that ;) That said, we agree on what should be done, and that's what matters ;) I just think that it might be time for you and Toad to decide to just stop responding to Opennet-related posts. It seems like the same arguments occur ad nauseum, which just wastes your time. My post was only trying to lay out my position to Jusa, as part of my explanation on why everyone has their own ideas of the best way forward. I did not intend reopen the debate. It's really tempting to reply to your post, since I disagree on the premises, but since it's a debate that's happened many times before I'll avoid wasting your time ;) -Colin From luke771 at gmail.com Fri Jun 8 13:31:17 2007 From: luke771 at gmail.com (Luke771) Date: Fri, 08 Jun 2007 15:31:17 +0200 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <823242bd0706071620m2b056ee7wc31b6de450f7bdd4@mail.gmail.com> References: <200706061911.33237.toad@amphibian.dyndns.org> <200706080005.46991.toad@amphibian.dyndns.org> <823242bd0706071620m2b056ee7wc31b6de450f7bdd4@mail.gmail.com> Message-ID: <46695A25.9070900@gmail.com> On bittorrent, I usually torify tracker connection but not p2p connections. That leaves me still vulnerable to my peers but makes it somewhat less easy to label my IP as 'evil file sharer' because monitoring the tracker won't show my real IP but a tor server. By the way, am I the only one thinking that PeerGuardian is useless because a 'real' bad guy's IP would never get on the shitlist? Oh, that's way OT. Never mind. (there @chat for that) Luke Ian Clarke wrote: > On 6/7/07, Matthew Toseland wrote: >> It doesn't matter >> if we have an opennet that outperforms BitTorrent over Tor > > On that subject, NEVER try using BitTorrent over Tor unless for some > reason you hate Tor and are trying to do a denial of service attack on > it. > > Ian. > From toad at amphibian.dyndns.org Fri Jun 8 14:55:38 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 8 Jun 2007 15:55:38 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <46695A25.9070900@gmail.com> References: <200706061911.33237.toad@amphibian.dyndns.org> <823242bd0706071620m2b056ee7wc31b6de450f7bdd4@mail.gmail.com> <46695A25.9070900@gmail.com> Message-ID: <200706081555.43874.toad@amphibian.dyndns.org> On Friday 08 June 2007 14:31, Luke771 wrote: > On bittorrent, I usually torify tracker connection but not p2p connections. > That leaves me still vulnerable to my peers but makes it somewhat less > easy to label my IP as 'evil file sharer' because monitoring the tracker > won't show my real IP but a tor server. Don't you have to tell the tracker your IP in order for it to co-ordinate transfers with other peers? Is your connection to the tracker encrypted? -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070608/cf4a8797/attachment.pgp From toad at amphibian.dyndns.org Fri Jun 8 15:14:30 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 8 Jun 2007 16:14:30 +0100 Subject: [freenet-dev] Interesting graph Message-ID: <200706081614.30769.toad@amphibian.dyndns.org> Snapshot from my node's stats page, with low uptime (a few minutes). Very obvious clustering around 0.2/0.3 and it tails off. IIRC later on it becomes more dispersed. Maybe this tells us something? Theory: We have a small number of long links to the wider network, and we swap with nodes beyond those links infrequently, because there is a low probability of a swap going through that link, but when it does, it gives us access to a wider range of nodes. Hence, the location distribution starts off very specialised, but over time becomes less so. Another explanation for it becoming less specialised over time would be newbie nodes with random locations. Any way to tell between the two theories? http://amphibian.dyndns.org/freenet/08-06-07-low-uptime-location-distributions.png -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070608/65e67db1/attachment.pgp From freenet-devl at david.sowder.com Fri Jun 8 15:45:10 2007 From: freenet-devl at david.sowder.com (David Sowder) Date: Fri, 08 Jun 2007 10:45:10 -0500 Subject: [freenet-dev] Interesting graph In-Reply-To: <200706081614.30769.toad@amphibian.dyndns.org> References: <200706081614.30769.toad@amphibian.dyndns.org> Message-ID: <46697986.9000807@david.sowder.com> Matthew Toseland wrote: > Snapshot from my node's stats page, with low uptime (a few minutes). Very > obvious clustering around 0.2/0.3 and it tails off. IIRC later on it becomes > more dispersed. Maybe this tells us something? Theory: We have a small number > of long links to the wider network, and we swap with nodes beyond those links > infrequently, because there is a low probability of a swap going through that > link, but when it does, it gives us access to a wider range of nodes. Hence, > the location distribution starts off very specialised, but over time becomes > less so. > > Another explanation for it becoming less specialised over time would be newbie > nodes with random locations. > > Any way to tell between the two theories? > > http://amphibian.dyndns.org/freenet/08-06-07-low-uptime-location-distributions.png > My two major nodes are seeing that clustering as well (and have seen clustering at that location, 0.2/0.3, for a number of weeks) and are part of the cluster (like your node is); one of my major nodes is peered with yours and the two used to be peered together a month or two ago, but I separated them to see if they'd drift apart; they haven't, but that may be because their neighborhoods are probably rather established now; a third node I brought up for testing from no peers and got about 10 peers via bot2bot has a wildly fluctuating location (probably because the number of connected peers fluctuates from 0 to 6 or so) is currently 0.48 (in fact it just moved moments ago from 0.52 or so). From toad at amphibian.dyndns.org Fri Jun 8 18:34:26 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 8 Jun 2007 19:34:26 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: <20070607084424.GE56932@tim.hack.org> References: <200706061911.33237.toad@amphibian.dyndns.org> <200706062344.33468.toad@amphibian.dyndns.org> <20070607084424.GE56932@tim.hack.org> Message-ID: <200706081934.30911.toad@amphibian.dyndns.org> Important detail: On a DNF, we accept the DNF's list of best-so-far-not-visited locations, but we limit the number of forks to say 2. Thus we limit the possible damage caused by a node deliberately returning locations far away from the target. On Thursday 07 June 2007 09:44, vive wrote: > On Wed, Jun 06, 2007 at 11:44:32PM +0100, Matthew Toseland wrote: > > On Wednesday 06 June 2007 22:19, vive wrote: > > > On Wed, Jun 06, 2007 at 09:43:52PM +0100, Matthew Toseland wrote: > > > > After a long discussion with vivee, new proposal: > > > > On a request, we should, in addition to HTL and best-so-far, track > > > > the 3 best locations of nodes we have seen, could have visited, but > > > > haven't. > > > > > > > > When we run out of HTL, this indicates that we are either a) lost in > > > > a pocket, or b) have actually reached the end of the request. We send > > > > a DataNotFound to our previous node, just as now. But the previous > > > > node can fork a request - provided that the node it would fork to is > > > > at least as close to the target than the 3rd-best not visited node. > > > > It could be one of the nodes we didn't visit, or it could be a new > > > > option because a node came out of swapping. > > > > > > Part of the idea to store the 3 best positions seen-so-far but not > > > *visited* (by the query taking greedy routing) is also to have 3 > > > chances of backtracking and continue at a good place (instead of just > > > backtracking and forking to a best-possible neighbor at a node as soon > > > as possible). > > > > > > If the backtrack does not hit a node with a neighbor closer than the > > > 3rd best position seen, then we backtrack until the query first reaches > > > any(*) of the 3 nodes that had the closest neighbors-seen-so-far. From > > > this node the query can continue to the neighbor that was not > > > previously visited (2nd-best for greedy routing, and next step should > > > have higher chances of leading to the destination). > > > > I don't see why we need this special case: We can simply fork if the fork > > is at least as closer to the target as the 3rd-best. This will normally > > result in 3 forks but it might be more or it might be less. > > What if a node that _seemed_ to be a good option has swapped position since > then? Lets say we use the rule of comparing positions against the 3rd best > seen. During backtracking we will no longer see that fork as a good one, > but we know it seemed so recently (perhaps it was even better than the path > selected, but a node could not receive traffic due to congestion or so). > That information may also be used - even if its not strictly up to date. > > One way to extend the rule of comparing positions would be to also attempt > querying in these directions that seemed good previously. But I agree on > that it's not necessary, only comparing positions may be the first way to > go and see how well it works. > > > > (*) If we take the first one encountered out of these three, then we > > > dont have to "un-backtrack" (e.g. go down the branch again that we > > > backtracked up from in the first case) if the query reaches *another* > > > dead end :-) > > > > > > > This should severely limit the amount of forking that happens after a > > > > normal completion, while still enabling us to backtrack our way out > > > > of rabbit holes. > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070608/bee16961/attachment.pgp From jargonautti at hotmail.com Fri Jun 8 20:06:47 2007 From: jargonautti at hotmail.com (Jusa Saari) Date: Fri, 08 Jun 2007 23:06:47 +0300 Subject: [freenet-dev] Getting stuck down rabbit holes References: <200706061911.33237.toad@amphibian.dyndns.org> <20070607204443.GA5511@freenetproject.org> <823242bd0706071434u191a7901g55ef6d5295024591@mail.gmail.com> <200706080006.43381.toad@amphibian.dyndns.org> <46694AC1.8060406@sq7.org> Message-ID: On Fri, 08 Jun 2007 08:25:37 -0400, Colin Davis wrote: > Jusa- I don't think this line of argument is productive. ;) They've had > this argument 10,000 times, and I don't think Matthew's position is going > to change. > > He fundamentally believes that ANY opennet system could be easily blocked > if the situation ever came to it. He has a good point, even if you don't > agree with him. It's much easier to block an opennet. He is, of course, right. It is much easier to find a node which advertises its existence than one which doesn't. That is obvious and not something I'm debating. What I _am_ debating is that, as is and for the foreseeable future, the only way to get into Freenet is to get noderefs from strangers in #freenet-refs, means that you can harvest most of the network by just listening to the IRC channel; as is, Freenet is operating as an opennet, just a very inconvenient one. This inconvenience does nothing useful, and in fact hinders both network adoption and structure as well as makes it more easily harvestable (by concentrating all the node ref exchanges into a single publically accessible channel). > Your point is that without a critical mass, Freenet is of limited > usefulness- It's hard to FIND friends who use Freenet, which makes a > global darknet difficult. This is also true. Of course, in order for you to have at least three friends (the minimum number of connections to get an actual network and not just a 1-dimensional node chain) out of your ten closest ones (tell to less close friends you're running Freenet, and you're essentially running a public node) about 3/10 of the population needs to be running Freenet. Somehow, I doubt this is going to happen, even if the figure is adjusted for things like people selecting like-minded friends. > I don't know that having this same discussion over and over will being up > and new data, or sway anyone ;) Yes, you are likely to be correct. Unfortunately, this means that Freenet will remain in obscurity with insignificant amount of users, since very few people will jump through the hoops to get it up and running - and even less people will spend the time to maintain it (by getting new connections as the old ones go down), especially given the very limited content at the time (mostly caused by those same insignificant user counts). Oh well, I guess it's better this way; we wouldn't want various oppressive regimens to think Freenet an actual threat to themselves and put a horse's head into Matthew's bed, now would we ?-) > > > > > I look at it, and know that opennet is necessary for the darknet to > prosper.. Once Opennet is in place, people have a larger incentive to run > Freenet. This means that they add permanent nodes, which is good. > > This build the larger base of Freenet users, which we need to make a > darknet work. Two opennet users can talk to one another offline, and agree > to exchange a Darknet Link. This grows the number of links that aren't > posted about anywhere. > > Later on, as new users join, they're friends already ARE using freenet. > This means that they can exchange darknet links with people they know, > without ever having to run opennet. > > > Opennet is the key to getting a the installed base which allows darknets > to work. > > -Colin > > > > > > > > > > > > > Jusa Saari wrote: >> On Fri, 08 Jun 2007 00:06:42 +0100, Matthew Toseland wrote: >> >>> Nonetheless the general thrust of Jusa's argument was that there is no >>> point having darknet. He's wrong. Opennet is a means to an end, not the >>> be all and end all. >> >> More to the point, I'm trying to argue that having a global darknet is >> not an attainable goal, and pushing for it will accomplish nothing >> except hinder Freenet growth and functionality. The chances for anyone >> wanting to join the network to have IRL friends already running it are >> nearly nonexistent due to the small amounts of such users currently in >> the network; and remember, you need at least three such contacts before >> your node is actually routing. >> >> And of course opennet is a means to an end. So is the whole Freenet, and >> every other computer program in existence, as well as the computer >> itself. >> >>> On Thursday 07 June 2007 22:34, Ian Clarke wrote: >>>> On 6/7/07, Florent Daigni?re >>>> wrote: >>>>> * Jusa Saari >>>>> [2007-06-07 23:23:48]: Implementing a workaround (opennet, >>>>> backtracking, ...) is only a way of fixing temporarily the topology >>>>> to the expense of both liberty (it has to be the default behaviour as >>>>> you pointed out) and safety (everyone knows that the opennet approach >>>>> has design caveats). >>>>> >>>>> "They that can give up essential liberty to obtain a little temporary >>>>> safety deserve neither liberty nor safety" -- Benjamin Franklin >>>> Your use of the Franklin quote suggests that you are looking at this >>>> backwards. >>>> >>>> The alternative here is not between users using an opennet or a >>>> darknet, its between them using an opennet, or another solution that >>>> is far worse (such as a public proxy). This is the simple reality of >>>> the situation that we see time and time again whenever we bother to >>>> listen to our users. >>>> >>>> Consequently, by giving people the option of an opennet, we aren't >>>> inviting them to give up liberty, we are inviting them to increase it. >>>> >>>> Most sane people think teenagers should have access to condoms, but >>>> some people think its a bad idea, claiming that giving teenagers >>>> access to condoms will encourage them to have sex. The point, >>>> obviously, is that teenagers will have sex anyway, the only question >>>> is whether it will be safe sex. >>>> >>>> Substitute opennet for condoms, and darknet for abstinence, and you >>>> see that many of those arguing against opennet are following the exact >>>> same wrongheaded line of reasoning as those that disagree with >>>> allowing teenagers access to condoms. >>>> >>>> Ian._______________________________________________ Devl mailing list >>> Devl at freenetproject.org >>> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl >> >> >> _______________________________________________ Devl mailing list >> Devl at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From toad at amphibian.dyndns.org Fri Jun 8 21:13:11 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Fri, 8 Jun 2007 22:13:11 +0100 Subject: [freenet-dev] Getting stuck down rabbit holes In-Reply-To: References: <200706061911.33237.toad@amphibian.dyndns.org> <46694AC1.8060406@sq7.org> Message-ID: <200706082213.16833.toad@amphibian.dyndns.org> On Friday 08 June 2007 21:06, Jusa Saari wrote: > > Your point is that without a critical mass, Freenet is of limited > > usefulness- It's hard to FIND friends who use Freenet, which makes a > > global darknet difficult. This is also true. > > Of course, in order for you to have at least three friends (the minimum > number of connections to get an actual network and not just a > 1-dimensional node chain) out of your ten closest ones (tell to less close > friends you're running Freenet, and you're essentially running a public > node) about 3/10 of the population needs to be running Freenet. Somehow, I > doubt this is going to happen, even if the figure is adjusted for things > like people selecting like-minded friends. Not true. There is absolutely nothing wrong with connecting to casual acquaintances. It will produce the correct topology, and it's much less dangerous than true opennet or #freenet-refs . > > I don't know that having this same discussion over and over will being up > > and new data, or sway anyone ;) > > Yes, you are likely to be correct. Unfortunately, this means that Freenet > will remain in obscurity with insignificant amount of users, since very > few people will jump through the hoops to get it up and running - and even > less people will spend the time to maintain it (by getting new connections > as the old ones go down), especially given the very limited content at the > time (mostly caused by those same insignificant user counts). Agreed with the first poster. However the tradeoff isn't that terrible - slightly more work for a much better initial connection, and if anyone you know IS on freenet already, big gains. > Oh well, I guess it's better this way; we wouldn't want various oppressive > regimens to think Freenet an actual threat to themselves and put a horse's > head into Matthew's bed, now would we ?-) We have good reason to believe that China has blocked Freenet 0.5's session bytes. -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070608/3830410c/attachment.pgp From nextgens at freenetproject.org Sat Jun 9 17:29:33 2007 From: nextgens at freenetproject.org (Florent =?iso-8859-1?Q?Daigni=E8re?=) Date: Sat, 9 Jun 2007 19:29:33 +0200 Subject: [freenet-dev] repository dump/reload Message-ID: <20070609172933.GB5509@freenetproject.org> Hi, On r13498 a file we don't have the right to re-distribute has been commited to the repository... As we haven't noticed it soon enough, I had to dump/reload the repository up to r13497... Some mails have been sent to the mailing lists but don't refer to anything [1][2] : please disregard them. People who have updated to r13499 in the meantime will have to re-do a full checkout.. I apologize about that. NextGen$ [1]http://archives.freenetproject.org/message/20070609.095819.28c3eccf.en.html [2]http://archives.freenetproject.org/message/20070609.163457.0322a6c3.en.html -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: Digital signature Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070609/dd9619b5/attachment.pgp From toad at amphibian.dyndns.org Mon Jun 11 16:51:51 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Mon, 11 Jun 2007 17:51:51 +0100 Subject: [freenet-dev] [freenet-cvs] r13509 - in trunk/freenet/src/freenet: clients/http l10n In-Reply-To: <20070610200110.D145247A3FE@emu.freenetproject.org> References: <20070610200110.D145247A3FE@emu.freenetproject.org> Message-ID: <200706111752.01207.toad@amphibian.dyndns.org> Is this necessary? On Sunday 10 June 2007 21:01, you wrote: > Author: zothar > Date: 2007-06-10 20:01:10 +0000 (Sun, 10 Jun 2007) > New Revision: 13509 > > Modified: > trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java > trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties > Log: > Use URL encoding in BookmarkEditorToadlet > > Modified: trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java > =================================================================== > --- > trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java 2007-06-0 >9 21:38:28 UTC (rev 13508) +++ > trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java 2007-06-1 >0 20:01:10 UTC (rev 13509) @@ -16,6 +16,9 @@ > import freenet.node.NodeClientCore; > import freenet.client.HighLevelSimpleClient; > import freenet.support.HTMLNode; > +import freenet.support.URLDecoder; > +import freenet.support.URLEncodedFormatException; > +import freenet.support.URLEncoder; > import freenet.support.api.HTTPRequest; > > public class BookmarkEditorToadlet extends Toadlet { > @@ -53,8 +56,8 @@ > > for(int i = 0; i < items.size(); i++) { > > - String itemPath = path + items.get(i).getName(); > - HTMLNode li = new HTMLNode("li", "class","item" , > items.get(i).getName()); + String itemPath = URLEncoder.encode(path + > items.get(i).getName()); + HTMLNode li = new HTMLNode("li", "class", > "item" , items.get(i).getName()); > > HTMLNode actions = new HTMLNode("span", "class", "actions"); > actions.addChild("a", "href", "?action=edit&bookmark=" + > itemPath).addChild("img", new String[] {"src", "alt", "title"}, new > String[] {"/static/icon/edit.png", edit, edit}); @@ -77,7 +80,7 @@ > BookmarkCategories cats = cat.getSubCategories(); > for(int i = 0; i < cats.size(); i++) { > > - String catPath = path + cats.get(i).getName() + "/"; > + String catPath = URLEncoder.encode(path + cats.get(i).getName() + "/"); > > HTMLNode subCat = list.addChild("li", "class", "cat", > cats.get(i).getName()); > > @@ -141,7 +144,15 @@ > > if (req.getParam("action").length() > 0 && > req.getParam("bookmark").length() > 0) { String action = > req.getParam("action"); > - String bookmarkPath = req.getParam("bookmark"); > + String bookmarkPath; > + try { > + bookmarkPath = URLDecoder.decode(req.getParam("bookmark"), false); > + } catch (URLEncodedFormatException e) { > + HTMLNode errorBox = > content.addChild(ctx.getPageMaker().getInfobox("infobox-error", error)); > + errorBox.addChild("#", > L10n.getString("BookmarkEditorToadlet.urlDecodeError")); > + this.writeReply(ctx, 200, "text/html", "OK", pageNode.generate()); > + return; > + } > Bookmark bookmark; > > if (bookmarkPath.endsWith("/")) > > Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties > =================================================================== > --- trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2007-06-09 > 21:38:28 UTC (rev 13508) +++ > trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2007-06-10 > 20:01:10 UTC (rev 13509) @@ -34,6 +34,7 @@ > BookmarkEditorToadlet.pasteTitle=Cut/Paste > BookmarkEditorToadlet.save=Save > BookmarkEditorToadlet.title=Bookmark Editor > +BookmarkEditorToadlet.urlDecodeError=URL Decode Error > BookmarkItem.bookmarkUpdated=The bookmarked site ${name} has been updated > to edition ${edition}. BookmarkItem.bookmarkUpdatedTitle=Bookmark Updated: > ${name} > BookmarkItem.bookmarkUpdatedWithLink=The bookmarked site > ${link}${name}${/link} has been updated to edition ${edition}. > > _______________________________________________ > cvs mailing list > cvs at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070611/b7a72c5d/attachment.pgp From freenet-devl at david.sowder.com Mon Jun 11 17:14:10 2007 From: freenet-devl at david.sowder.com (David Sowder) Date: Mon, 11 Jun 2007 12:14:10 -0500 Subject: [freenet-dev] [freenet-cvs] r13509 - in trunk/freenet/src/freenet: clients/http l10n In-Reply-To: <200706111752.01207.toad@amphibian.dyndns.org> References: <20070610200110.D145247A3FE@emu.freenetproject.org> <200706111752.01207.toad@amphibian.dyndns.org> Message-ID: <466D82E2.7050106@david.sowder.com> Matthew Toseland wrote: > Is this necessary? > Yes. I managed to create a bookmark I couldn't manage later. It contained a "&" character. > On Sunday 10 June 2007 21:01, you wrote: > >> Author: zothar >> Date: 2007-06-10 20:01:10 +0000 (Sun, 10 Jun 2007) >> New Revision: 13509 >> >> Modified: >> trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java >> trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties >> Log: >> Use URL encoding in BookmarkEditorToadlet >> >> Modified: trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java >> =================================================================== >> --- >> trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java 2007-06-0 >> 9 21:38:28 UTC (rev 13508) +++ >> trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java 2007-06-1 >> 0 20:01:10 UTC (rev 13509) @@ -16,6 +16,9 @@ >> import freenet.node.NodeClientCore; >> import freenet.client.HighLevelSimpleClient; >> import freenet.support.HTMLNode; >> +import freenet.support.URLDecoder; >> +import freenet.support.URLEncodedFormatException; >> +import freenet.support.URLEncoder; >> import freenet.support.api.HTTPRequest; >> >> public class BookmarkEditorToadlet extends Toadlet { >> @@ -53,8 +56,8 @@ >> >> for(int i = 0; i < items.size(); i++) { >> >> - String itemPath = path + items.get(i).getName(); >> - HTMLNode li = new HTMLNode("li", "class","item" , >> items.get(i).getName()); + String itemPath = URLEncoder.encode(path + >> items.get(i).getName()); + HTMLNode li = new HTMLNode("li", "class", >> "item" , items.get(i).getName()); >> >> HTMLNode actions = new HTMLNode("span", "class", "actions"); >> actions.addChild("a", "href", "?action=edit&bookmark=" + >> itemPath).addChild("img", new String[] {"src", "alt", "title"}, new >> String[] {"/static/icon/edit.png", edit, edit}); @@ -77,7 +80,7 @@ >> BookmarkCategories cats = cat.getSubCategories(); >> for(int i = 0; i < cats.size(); i++) { >> >> - String catPath = path + cats.get(i).getName() + "/"; >> + String catPath = URLEncoder.encode(path + cats.get(i).getName() + "/"); >> >> HTMLNode subCat = list.addChild("li", "class", "cat", >> cats.get(i).getName()); >> >> @@ -141,7 +144,15 @@ >> >> if (req.getParam("action").length() > 0 && >> req.getParam("bookmark").length() > 0) { String action = >> req.getParam("action"); >> - String bookmarkPath = req.getParam("bookmark"); >> + String bookmarkPath; >> + try { >> + bookmarkPath = URLDecoder.decode(req.getParam("bookmark"), false); >> + } catch (URLEncodedFormatException e) { >> + HTMLNode errorBox = >> content.addChild(ctx.getPageMaker().getInfobox("infobox-error", error)); >> + errorBox.addChild("#", >> L10n.getString("BookmarkEditorToadlet.urlDecodeError")); >> + this.writeReply(ctx, 200, "text/html", "OK", pageNode.generate()); >> + return; >> + } >> Bookmark bookmark; >> >> if (bookmarkPath.endsWith("/")) >> >> Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties >> =================================================================== >> --- trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2007-06-09 >> 21:38:28 UTC (rev 13508) +++ >> trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2007-06-10 >> 20:01:10 UTC (rev 13509) @@ -34,6 +34,7 @@ >> BookmarkEditorToadlet.pasteTitle=Cut/Paste >> BookmarkEditorToadlet.save=Save >> BookmarkEditorToadlet.title=Bookmark Editor >> +BookmarkEditorToadlet.urlDecodeError=URL Decode Error >> BookmarkItem.bookmarkUpdated=The bookmarked site ${name} has been updated >> to edition ${edition}. BookmarkItem.bookmarkUpdatedTitle=Bookmark Updated: >> ${name} >> BookmarkItem.bookmarkUpdatedWithLink=The bookmarked site >> ${link}${name}${/link} has been updated to edition ${edition}. >> >> _______________________________________________ >> cvs mailing list >> cvs at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs >> >> ------------------------------------------------------------------------ >> >> _______________________________________________ >> Devl mailing list >> Devl at freenetproject.org >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl From toad at amphibian.dyndns.org Mon Jun 11 17:17:35 2007 From: toad at amphibian.dyndns.org (Matthew Toseland) Date: Mon, 11 Jun 2007 18:17:35 +0100 Subject: [freenet-dev] =?iso-8859-1?q?=5Bfreenet-cvs=5D_r13509_-_in=09trun?= =?iso-8859-1?q?k/freenet/src/freenet=3A_clients/http_l10n?= In-Reply-To: <466D82E2.7050106@david.sowder.com> References: <20070610200110.D145247A3FE@emu.freenetproject.org> <200706111752.01207.toad@amphibian.dyndns.org> <466D82E2.7050106@david.sowder.com> Message-ID: <200706111817.40967.toad@amphibian.dyndns.org> On Monday 11 June 2007 18:14, David Sowder wrote: > Matthew Toseland wrote: > > Is this necessary? > > Yes. I managed to create a bookmark I couldn't manage later. It > contained a "&" character. I don't understand why. Anything that goes through HTMLNode is automatically HTMLEncode'd. > > > On Sunday 10 June 2007 21:01, you wrote: > >> Author: zothar > >> Date: 2007-06-10 20:01:10 +0000 (Sun, 10 Jun 2007) > >> New Revision: 13509 > >> > >> Modified: > >> trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java > >> trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties > >> Log: > >> Use URL encoding in BookmarkEditorToadlet > >> > >> Modified: > >> trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java > >> =================================================================== --- > >> trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java 2007-0 > >>6-0 9 21:38:28 UTC (rev 13508) +++ > >> trunk/freenet/src/freenet/clients/http/BookmarkEditorToadlet.java 2007-0 > >>6-1 0 20:01:10 UTC (rev 13509) @@ -16,6 +16,9 @@ > >> import freenet.node.NodeClientCore; > >> import freenet.client.HighLevelSimpleClient; > >> import freenet.support.HTMLNode; > >> +import freenet.support.URLDecoder; > >> +import freenet.support.URLEncodedFormatException; > >> +import freenet.support.URLEncoder; > >> import freenet.support.api.HTTPRequest; > >> > >> public class BookmarkEditorToadlet extends Toadlet { > >> @@ -53,8 +56,8 @@ > >> > >> for(int i = 0; i < items.size(); i++) { > >> > >> - String itemPath = path + items.get(i).getName(); > >> - HTMLNode li = new HTMLNode("li", "class","item" , > >> items.get(i).getName()); + String itemPath = URLEncoder.encode(path + > >> items.get(i).getName()); + HTMLNode li = new HTMLNode("li", "class", > >> "item" , items.get(i).getName()); > >> > >> HTMLNode actions = new HTMLNode("span", "class", "actions"); > >> actions.addChild("a", "href", "?action=edit&bookmark=" + > >> itemPath).addChild("img", new String[] {"src", "alt", "title"}, new > >> String[] {"/static/icon/edit.png", edit, edit}); @@ -77,7 +80,7 @@ > >> BookmarkCategories cats = cat.getSubCategories(); > >> for(int i = 0; i < cats.size(); i++) { > >> > >> - String catPath = path + cats.get(i).getName() + "/"; > >> + String catPath = URLEncoder.encode(path + cats.get(i).getName() + > >> "/"); > >> > >> HTMLNode subCat = list.addChild("li", "class", "cat", > >> cats.get(i).getName()); > >> > >> @@ -141,7 +144,15 @@ > >> > >> if (req.getParam("action").length() > 0 && > >> req.getParam("bookmark").length() > 0) { String action = > >> req.getParam("action"); > >> - String bookmarkPath = req.getParam("bookmark"); > >> + String bookmarkPath; > >> + try { > >> + bookmarkPath = URLDecoder.decode(req.getParam("bookmark"), false); > >> + } catch (URLEncodedFormatException e) { > >> + HTMLNode errorBox = > >> content.addChild(ctx.getPageMaker().getInfobox("infobox-error", error)); > >> + errorBox.addChild("#", > >> L10n.getString("BookmarkEditorToadlet.urlDecodeError")); > >> + this.writeReply(ctx, 200, "text/html", "OK", pageNode.generate()); > >> + return; > >> + } > >> Bookmark bookmark; > >> > >> if (bookmarkPath.endsWith("/")) > >> > >> Modified: trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties > >> =================================================================== > >> --- trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2007-06-09 > >> 21:38:28 UTC (rev 13508) +++ > >> trunk/freenet/src/freenet/l10n/freenet.l10n.en.properties 2007-06-10 > >> 20:01:10 UTC (rev 13509) @@ -34,6 +34,7 @@ > >> BookmarkEditorToadlet.pasteTitle=Cut/Paste > >> BookmarkEditorToadlet.save=Save > >> BookmarkEditorToadlet.title=Bookmark Editor > >> +BookmarkEditorToadlet.urlDecodeError=URL Decode Error > >> BookmarkItem.bookmarkUpdated=The bookmarked site ${name} has been > >> updated to edition ${edition}. > >> BookmarkItem.bookmarkUpdatedTitle=Bookmark Updated: ${name} > >> BookmarkItem.bookmarkUpdatedWithLink=The bookmarked site > >> ${link}${name}${/link} has been updated to edition ${edition}. > >> > >> _______________________________________________ > >> cvs mailing list > >> cvs at freenetproject.org > >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/cvs > >> > >> ------------------------------------------------------------------------ > >> > >> _______________________________________________ > >> Devl mailing list > >> Devl at freenetproject.org > >> http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl > > _______________________________________________ > Devl mailing list > Devl at freenetproject.org > http://emu.freenetproject.org/cgi-bin/mailman/listinfo/devl -------------- next part -------------- A non-text attachment was scrubbed... Name: not available Type: application/pgp-signature Size: 189 bytes Desc: not available Url : http://emu.freenetproject.org/pipermail/devl/attachments/20070611/5ea74a86/attachment.pgp From juiceman69 at gmail.com Mon Jun 11 17:39:10 2007 From: juiceman69 at gmail.com (Juiceman) Date: Mon, 11 Jun 2007 13:39:10 -0400 Subject: [freen