From zothar at freenetproject.org Fri Nov 2 03:10:17 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Fri, 2 Nov 2007 03:10:17 +0000 (UTC) Subject: [Pyfreenet] r15660 - trunk/apps/pyFreenet Message-ID: <20071102031017.D2B37479856@freenetproject.org> Author: zothar Date: 2007-11-02 03:10:17 +0000 (Fri, 02 Nov 2007) New Revision: 15660 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: Always disable darknet ref trading during the first-time configuration creation run. Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-02 01:08:44 UTC (rev 15659) +++ trunk/apps/pyFreenet/refbot.py 2007-11-02 03:10:17 UTC (rev 15660) @@ -914,7 +914,8 @@ self.setup_bot2bot_trades_only( opts ) opts['refurl'] = ''; opts['opennet_refurl'] = ''; - self.setup_darknet_trades( opts ) + #self.setup_darknet_trades( opts ) + opts['darknet_trades'] = 'n'; self.setup_opennet_trades( opts ) self.setup_refurl( opts ) self.setup_opennet_refurl( opts ) From zothar at freenetproject.org Fri Nov 2 23:02:33 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Fri, 2 Nov 2007 23:02:33 +0000 (UTC) Subject: [Pyfreenet] r15662 - trunk/apps/pyFreenet Message-ID: <20071102230233.2F4983909BC@freenetproject.org> Author: zothar Date: 2007-11-02 23:02:32 +0000 (Fri, 02 Nov 2007) New Revision: 15662 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: Die if we have too many node communication failures in a row. Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-02 10:02:41 UTC (rev 15661) +++ trunk/apps/pyFreenet/refbot.py 2007-11-02 23:02:32 UTC (rev 15662) @@ -835,6 +835,8 @@ self.sendlock = threading.Lock() self.adderThreads = [] + self.directOrDieLock = threading.Lock() + self.nodeCommsProblemCount = 0; self.identityCheckerThreads = [] self.peerUpdaterThreads = [] self.peer_update_interval = 60 @@ -914,11 +916,11 @@ self.setup_bot2bot_trades_only( opts ) opts['refurl'] = ''; opts['opennet_refurl'] = ''; + self.setup_opennet_trades( opts ) + opts['darknet_trades'] = 'n'; #self.setup_darknet_trades( opts ) - opts['darknet_trades'] = 'n'; - self.setup_opennet_trades( opts ) + self.setup_opennet_refurl( opts ) self.setup_refurl( opts ) - self.setup_opennet_refurl( opts ) self.setup_privmsg_only( opts ) opts['greetinterval'] = 1800 @@ -990,7 +992,7 @@ """ """ while 1: - opts['darknet_trades'] = self.prompt("Should we trade darknet refs?", "y") + opts['darknet_trades'] = self.prompt("Should we trade darknet refs?", "n") opts['darknet_trades'] = opts['darknet_trades'].lower(); if( opts['darknet_trades'] in [ 'y', 'n' ] ): break; @@ -1752,6 +1754,39 @@ return True #@-node:check_ref_url_and_complain + #@+node:dodirectordie + def dodirectordie(self, peernick ): + + #log( "** dodirectordie(): directOrDieLock.acquire() before processing peernick: %s" % ( peernick )); + self.directOrDieLock.acquire( 1 ); + f = None; + nodeIsUp = False; + try: + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port ) + nodeIsUp = True; + f.shutdown(); + self.nodeCommsProblemCount = 0; + except Exception, msg: + nodeIsUp = False; + if(f != None): + f.shutdown(); + #log( "** dodirectordie(): directOrDieLock.release() after processing peernick: %s" % ( peernick )); + self.directOrDieLock.release(); + if( nodeIsUp ): + self.privmsg( peernick, "I can talk to my node" ) + else: + self.privmsg( peernick, "I cannot talk to my node" ) + self.privmsg( + self.channel, + "Sorry, seems I couldn't collect my target number of refs before the node stopped talking to me. Thanks anyway." + ) + self.privmsg( + self.channel, + "Bye" + ) + self.after(4, self.die) + + #@-node:dodirectordie #@+node:has_ref def has_ref(self, url): """ @@ -1830,9 +1865,11 @@ if( self.check_bot_peer_is_already_added( botNick )): continue if(1 == status): + self.nodeCommsProblemCount = 0; self.privmsg( botNick, "havepeer" ); self.bots[ botNick ][ "already_added" ] = True; elif( 0 == status ): + self.nodeCommsProblemCount = 0; if( self.bots[ botNick ].has_key( "ref" ) and self.bots[ botNick ].has_key( "ref_terminated" ) and self.bots[ botNick ].has_key( "ref_is_good" )): self.privmsg( botNick, "haveref" ); elif( self.bots[ botNick ].has_key( "ref" )): @@ -1852,9 +1889,11 @@ if( self.check_bot_peer_is_already_added( botNick )): continue if(1 == status): + self.nodeCommsProblemCount = 0; self.privmsg( botNick, "haveopennetpeer" ); self.bots[ botNick ][ "opennet_already_added" ] = True; elif( 0 == status ): + self.nodeCommsProblemCount = 0; if( self.bots[ botNick ].has_key( "opennet_ref" ) and self.bots[ botNick ].has_key( "opennet_ref_terminated" ) and self.bots[ botNick ].has_key( "opennet_ref_is_good" )): self.privmsg( botNick, "haveopennetref" ); elif( self.bots[ botNick ].has_key( "opennet_ref" )): @@ -1879,6 +1918,7 @@ log("adderThread has status: %s url: %s error_msg: %s" % (adderThread.status, adderThread.url, adderThread.error_msg)) self.adderThreads.remove(adderThread) if(0 < adderThread.status): + self.nodeCommsProblemCount = 0; if( adderThread.peerRef != None ): if( adderThread.botAddType == "request" ): if(adderThread.isDarknetRef): @@ -1954,6 +1994,7 @@ error_str = "there was a problem fetching the given URL. Please correct the URL <%s> and try again, try again later or perhaps try a different pastebin such as %s" % (adderThread.url, known_pastebin_result) elif(-3 == adderThread.status): error_str = "there was a problem talking to the node. Please try again later." + self.nodeCommsProblemCount += 1; elif(-4 == adderThread.status): error_str = "the node reports that it already has a peer with that identity. Ref not re-added." elif(-5 == adderThread.status): @@ -1976,6 +2017,17 @@ refs_plural_str = "s" refs_to_go_str = " (%d ref%s to go)" % ( refs_to_go, refs_plural_str ) adderThread.replyfunc("%s%s" % (error_str, refs_to_go_str)) + if( 2 <= self.nodeCommsProblemCount ): + # Die if we've had too many failures talking to the node + self.privmsg( + self.channel, + "Sorry, seems I couldn't collect my target number of refs before the node stopped talking to me. Thanks anyway." + ) + self.privmsg( + self.channel, + "Bye" + ) + self.after(4, self.die) self.after(0.5, self.process_any_refs_added) #@-node:process_any_refs_added @@ -2240,6 +2292,14 @@ self.after(random.randint(7, 20), self.bot.sendGetOptions, self.peernick) # Ask for their options after 7-20 seconds #@-node:cmd_bothello + #@+node:cmd_dodirectordie + def cmd_dodirectordie(self, replyfunc, is_from_privmsg, args): + + if( not is_from_privmsg ): + return; + self.bot.dodirectordie(self.peernick) + + #@-node:cmd_dodirectordie #@+node:cmd_doopennetrefswapallow def cmd_doopennetrefswapallow(self, replyfunc, is_from_privmsg, args): # NOTE: We'll not have asked if from our perspective we didn't want to swap, but we don't want to add a opennet ref for a bot we don't think we can respond to (because they disconnected or something) From zothar at freenetproject.org Sun Nov 4 17:11:11 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Sun, 4 Nov 2007 17:11:11 +0000 (UTC) Subject: [Pyfreenet] r15682 - trunk/apps/pyFreenet/fcp Message-ID: <20071104171111.E53E54796EF@freenetproject.org> Author: zothar Date: 2007-11-04 17:11:11 +0000 (Sun, 04 Nov 2007) New Revision: 15682 Modified: trunk/apps/pyFreenet/fcp/node.py Log: pyFreenet: add support for socket timeouts and getVerbosity() Modified: trunk/apps/pyFreenet/fcp/node.py =================================================================== --- trunk/apps/pyFreenet/fcp/node.py 2007-11-04 10:57:04 UTC (rev 15681) +++ trunk/apps/pyFreenet/fcp/node.py 2007-11-04 17:11:11 UTC (rev 15682) @@ -227,6 +227,8 @@ should be written, defaults to stdout - verbosity - how detailed the log messages should be, defaults to 0 (silence) + - socketTimeout - value to pass to socket object's settimeout() if + available and the value is not None, defaults to None Attributes of interest: - jobs - a dict of currently running jobs (persistent and nonpersistent). @@ -252,6 +254,7 @@ self.host = kw.get('host', env.get("FCP_HOST", defaultFCPHost)) self.port = kw.get('port', env.get("FCP_PORT", defaultFCPPort)) self.port = int(self.port) + self.socketTimeout = kw.get('socketTimeout', None) # set up the logger logfile = kw.get('logfile', None) or sys.stdout @@ -265,6 +268,12 @@ # try to connect to node self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + if(None != self.socketTimeout): + try: + self.socket.settimeout(self.socketTimeout) + except Exception, e: + # Socket timeout setting is not available until Python 2.3, so ignore exceptions + pass try: self.socket.connect((self.host, self.port)) except Exception, e: @@ -1767,6 +1776,41 @@ Identifier=id, Global=True, async=True, waituntilsent=True) #@-node:clearGlobalJob + #@+node:setSocketTimeout + def getSocketTimeout(self): + """ + Gets the socketTimeout for future socket calls; + returns None if not supported by Python version + """ + try: + return self.socket.gettimeout() + except Exception, e: + # Socket timeout setting is not available until Python 2.3, so ignore exceptions + pass + return None + + #@-node:setSocketTimeout + #@+node:setSocketTimeout + def setSocketTimeout(self, socketTimeout): + """ + Sets the socketTimeout for future socket calls + """ + self.socketTimeout = socketTimeout + try: + self.socket.settimeout(self.socketTimeout) + except Exception, e: + # Socket timeout setting is not available until Python 2.3, so ignore exceptions + pass + + #@-node:setSocketTimeout + #@+node:getVerbosity + def getVerbosity(self): + """ + Gets the verbosity for future logging calls + """ + return self.verbosity + + #@-node:getVerbosity #@+node:setVerbosity def setVerbosity(self, verbosity): """ From zothar at freenetproject.org Sun Nov 4 22:50:33 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Sun, 4 Nov 2007 22:50:33 +0000 (UTC) Subject: [Pyfreenet] r15687 - trunk/apps/pyFreenet Message-ID: <20071104225033.0634A4797DC@freenetproject.org> Author: zothar Date: 2007-11-04 22:50:32 +0000 (Sun, 04 Nov 2007) New Revision: 15687 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: bots on a channel now collectively track a list of user nicks seen on the channel recently, to later be used in direct cooperative bot announcements on new nick channel join. Because the new code is essentially in beta test, this release expires in six hours rather than the normal one week. There were also minor/miscellaneous unrelated tweaks and fixes. Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-04 19:33:58 UTC (rev 15686) +++ trunk/apps/pyFreenet/refbot.py 2007-11-04 22:50:32 UTC (rev 15687) @@ -89,6 +89,8 @@ self.botAnnouncePool = [] self.botDarknetIdentities = {} self.botOpennetIdentities = {} + self.botTimeWhenStarted = time.time(); + self.seenChannelUsers = [] # check that we've got a revision capable fcp/node.py (must be before using it) try: @@ -123,9 +125,14 @@ hour_seconds = 60 * minute_seconds; day_seconds = 24 * hour_seconds; week_seconds = 7 * day_seconds; - if( last_version_file_age > ( 1 * week_seconds )): + #if( last_version_file_age > ( 1 * week_seconds )): + # log("***"); + # log("*** This release of the refbot is more than one week old. Please run updater.py and then try starting refbot.py again."); + # log("***"); + # my_exit( 1 ); + if( last_version_file_age > ( 6 * hour_seconds )): log("***"); - log("*** This release of the refbot is more than one week old. Please run updater.py and try again."); + log("*** This release of the refbot (under testing) is more than six hours old. Please run updater.py and then start refbot.py again."); log("***"); my_exit( 1 ); @@ -438,6 +445,7 @@ if( self.bot2bot_announces_enabled ): self.botAnnouncePool.append( self.botircnick ); + self.seenChannelUsers.append( self.botircnick ); # finally construct the parent MiniBot.__init__(self, **kw) @@ -1170,6 +1178,10 @@ self.after(1, self.process_any_identities_checked) self.after(0.5, self.process_any_refs_added) self.after(1, self.process_peer_updates) + + for user in self.usersInChan: + if( not user in self.seenChannelUsers ): + self.seenChannelUsers.append( user ); log("****** on_ready") @@ -1179,8 +1191,51 @@ """ When another user (or us) have joined (post processing by inheriting class) """ - # We don't know if it's a bot at this point - pass + # NOTE: We don't know if it's a bot at this point + log("** DEBUG: post_on_join() called with sender: %s target: %s" % ( sender, target )); + log("** DEBUG: self.usersInChan: %s" % ( self.usersInChan )); + if( not sender in self.seenChannelUsers ): + maxSeenChannelUsersCount = len( self.usersInChan ) + 50; + log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); + while( maxSeenChannelUsersCount < len( self.seenChannelUsers )): + self.seenChannelUsers.pop( 0 ); + log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); + self.seenChannelUsers.append( sender ); + log("** DEBUG: self.seenChannelUsers: %s" % ( self.seenChannelUsers )); + log("** DEBUG: self.botAnnouncePool: %s" % ( self.botAnnouncePool )); + botInstanceAge = time.time() - self.botTimeWhenStarted; + log("** DEBUG: botInstanceAge: %s seconds" % ( botInstanceAge )); + # We won't announce if we've only been up for less than two minutes and we're not likely to know the whole botAnnouncePool yet + if( botInstanceAge > 120 ): + # + # + # **FIXME** The following is temporary during cooperative bot announce infrastructure testing + # + if( "_bot" != sender[ -4: ].lower() and self.botircnick == self.botAnnouncePool[ 0 ] ): + # **FIXME** This code will be replaced by the "selected announcer's" announcement private message to the joining user + log("** DEBUG: would welcome new, unseen-by-bot channel user here"); + if( "Zothar" in self.usersInChan ): + self.privmsg( + "Zothar", + "Would welcome new, unseen-by-bot channel user here: %s botAnnouncePool: %s" % ( sender, self.botAnnouncePool[ :3 ] ) + ) + elif( "Zothar_Work" in self.usersInChan ): + self.privmsg( + "Zothar_Work", + "Would welcome new, unseen-by-bot channel user here: %s botAnnouncePool: %s" % ( sender, self.botAnnouncePool[ :3 ] ) + ) + elif( "_bot" != sender[ -4: ].lower() and self.botircnick == "Zothar70d_bot" ): + # **FIXME** This code will be removed once cooperative bot announce testing is complete + if( "Zothar" in self.usersInChan ): + self.privmsg( + "Zothar", + "%s sould welcome new, unseen-by-bot channel user here: %s botAnnouncePool: %s" % ( self.botAnnouncePool[ 0 ], sender, self.botAnnouncePool[ :3 ] ) + ) + elif( "Zothar_Work" in self.usersInChan ): + self.privmsg( + "Zothar_Work", + "%s sould welcome new, unseen-by-bot channel user here: %s botAnnouncePool: %s" % ( self.botAnnouncePool[ 0 ], sender, self.botAnnouncePool[ :3 ] ) + ) #@-node:post_on_join #@+node:post_on_nick @@ -1188,6 +1243,7 @@ """ When another user (or us) have changed nicks (post processing by inheriting class) """ + log("** DEBUG: self.usersInChan: %s" % ( self.usersInChan )); if(self.bots.has_key( sender )): bot_data = self.bots[ sender ] del self.bots[ sender ] @@ -1195,6 +1251,7 @@ if( sender in self.botAnnouncePool ): k = self.botAnnouncePool.index( sender ); self.botAnnouncePool[ k ] = target; + self.botAnnouncePool.sort(); log("** bots: %s" % ( self.bots.keys() )) #@-node:post_on_nick @@ -1203,6 +1260,7 @@ """ When another user (or us) have left a channel (post processing by inheriting class) """ + log("** DEBUG: self.usersInChan: %s" % ( self.usersInChan )); if(self.bots.has_key( sender )): if( sender in self.botAnnouncePool ): self.botAnnouncePool.remove( sender ); @@ -1215,6 +1273,7 @@ """ When another user (or us) have quit a server (post processing by inheriting class) """ + log("** DEBUG: self.usersInChan: %s" % ( self.usersInChan )); if(self.bots.has_key( sender )): if( self.bots[ sender ].has_key( "identity" )): identity = self.bots[ sender ][ "identity" ] @@ -1238,6 +1297,8 @@ #@+node:addBotIdentity def addBotIdentity(self, botNick, botDarknetIdentity ): + if( "None" == botDarknetIdentity ): + return False; if( self.botDarknetIdentities.has_key( botDarknetIdentity )): return False; if( not self.bots.has_key( botNick )): @@ -1252,6 +1313,8 @@ #@+node:addBotOpennetIdentity def addBotOpennetIdentity(self, botNick, botOpennetIdentity ): + if( "None" == botOpennetIdentity ): + return False; if( self.botOpennetIdentities.has_key( botOpennetIdentity )): return False; if( not self.bots.has_key( botNick )): @@ -1263,6 +1326,19 @@ return True; #@-node:addBotOpennetIdentity + #@+node:addSeenChannelUsers + def addSeenChannelUsers(self, botNick, botSeenChannelUsers ): + + if(self.bots.has_key( botNick )): + try: + seenChannelUsers = eval( botSeenChannelUsers ) + except: + return + for user in seenChannelUsers: + if( not user in self.seenChannelUsers ): + self.seenChannelUsers.append( user ); + + #@-node:addSeenChannelUsers #@+node:getPeerUpdate def getPeerUpdate(self): @@ -1485,6 +1561,14 @@ self.privmsg( target, "myoptions %s" % ( self.api_options )) #@-node:sendMyOptions + #@+node:sendSeenChannelUsers + def sendSeenChannelUsers(self, target): + """ + Give them our seenChannelUsers + """ + self.privmsg( target, "myseenchannelusers %s" % ( self.seenChannelUsers )) + + #@-node:sendSeenChannelUsers #@+node:setPeerBotOptions def setPeerBotOptions(self, botNick, botOptions ): @@ -1498,6 +1582,7 @@ if( botNick not in self.botAnnouncePool ): self.botAnnouncePool.append( botNick ); self.botAnnouncePool.sort(); + self.after( 3, self.sendSeenChannelUsers, botNick ); #@-node:setPeerBotOptions #@+node:spamChannel @@ -2531,6 +2616,12 @@ self.bot.sendrefdirect( self.peernick, self.bot.bots.has_key( self.peernick )); #@-node:cmd_getrefdirect + #@+node:cmd_getseenchannelusers + def cmd_getseenchannelusers(self, replyfunc, is_from_privmsg, args): + + self.bot.sendSeenChannelUsers( self.peernick ) + + #@-node:cmd_getseenchannelusers #@+node:cmd_haveopennetpeer def cmd_haveopennetpeer(self, replyfunc, is_from_privmsg, args): if( 1 == len( args ) and self.bot.bots.has_key( self.peernick )): @@ -2652,6 +2743,14 @@ self.after(random.randint(7, 20), self.bot.sendGetOpennetIdentity, self.peernick) # Ask for their opennet identity after 7-20 seconds #@-node:cmd_myoptions + #@+node:cmd_myseenchannelusers + def cmd_myseenchannelusers(self, replyfunc, is_from_privmsg, args): + + args = string.join( args, " " ); + if( self.bot.bots.has_key( self.peernick )): + self.bot.addSeenChannelUsers( self.peernick, args ); + + #@-node:cmd_myseenchannelusers #@+node:cmd_opennetIdentity def cmd_opennetIdentity(self, replyfunc, is_from_privmsg, args): @@ -2670,7 +2769,7 @@ self.bot.bots[ self.peernick ][ "opennet_ref" ] = [] self.bot.bots[ self.peernick ][ "opennet_ref" ].append( peerRefLine ) if("end" == peerRefLine.lower()): - # **FIXME** Perhaps a check for getting a darknet ref instaed of an opennet ref + # **FIXME** Perhaps a check for getting a darknet ref instead of an opennet ref self.bot.bots[ self.peernick ][ "opennet_ref_terminated" ] = True self.bot.check_opennet_ref_from_bot_and_act( self.peernick ) @@ -2693,7 +2792,7 @@ self.bot.bots[ self.peernick ][ "ref" ] = [] self.bot.bots[ self.peernick ][ "ref" ].append( peerRefLine ) if("end" == peerRefLine.lower()): - # **FIXME** Perhaps a check for getting a opennet ref instaed of an darknet ref + # **FIXME** Perhaps a check for getting a opennet ref instead of an darknet ref self.bot.bots[ self.peernick ][ "ref_terminated" ] = True self.bot.check_darknet_ref_from_bot_and_act( self.peernick ) From zothar at freenetproject.org Sun Nov 4 23:41:32 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Sun, 4 Nov 2007 23:41:32 +0000 (UTC) Subject: [Pyfreenet] r15692 - trunk/apps/pyFreenet Message-ID: <20071104234132.50E584797DC@freenetproject.org> Author: zothar Date: 2007-11-04 23:41:32 +0000 (Sun, 04 Nov 2007) New Revision: 15692 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: clarify the message talking about a problem talking to the partner node Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-04 23:37:31 UTC (rev 15691) +++ trunk/apps/pyFreenet/refbot.py 2007-11-04 23:41:32 UTC (rev 15692) @@ -2078,7 +2078,7 @@ else: error_str = "there was a problem fetching the given URL. Please correct the URL <%s> and try again, try again later or perhaps try a different pastebin such as %s" % (adderThread.url, known_pastebin_result) elif(-3 == adderThread.status): - error_str = "there was a problem talking to the node. Please try again later." + error_str = "there was a problem while I was talking to my partner node. Please try again later." self.nodeCommsProblemCount += 1; elif(-4 == adderThread.status): error_str = "the node reports that it already has a peer with that identity. Ref not re-added." From zothar at freenetproject.org Mon Nov 5 00:23:36 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Mon, 5 Nov 2007 00:23:36 +0000 (UTC) Subject: [Pyfreenet] r15693 - trunk/apps/pyFreenet Message-ID: <20071105002336.1642D47A37A@freenetproject.org> Author: zothar Date: 2007-11-05 00:23:35 +0000 (Mon, 05 Nov 2007) New Revision: 15693 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: report details of problems talking to the node to Zothar, if he's around, so that we can see what types of things cause that problem as they may not all be socket timeout type issues Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-04 23:41:32 UTC (rev 15692) +++ trunk/apps/pyFreenet/refbot.py 2007-11-05 00:23:35 UTC (rev 15693) @@ -1229,12 +1229,12 @@ if( "Zothar" in self.usersInChan ): self.privmsg( "Zothar", - "%s sould welcome new, unseen-by-bot channel user here: %s botAnnouncePool: %s" % ( self.botAnnouncePool[ 0 ], sender, self.botAnnouncePool[ :3 ] ) + "%s should welcome new, unseen-by-bot channel user here: %s botAnnouncePool: %s" % ( self.botAnnouncePool[ 0 ], sender, self.botAnnouncePool[ :3 ] ) ) elif( "Zothar_Work" in self.usersInChan ): self.privmsg( "Zothar_Work", - "%s sould welcome new, unseen-by-bot channel user here: %s botAnnouncePool: %s" % ( self.botAnnouncePool[ 0 ], sender, self.botAnnouncePool[ :3 ] ) + "%s should welcome new, unseen-by-bot channel user here: %s botAnnouncePool: %s" % ( self.botAnnouncePool[ 0 ], sender, self.botAnnouncePool[ :3 ] ) ) #@-node:post_on_join @@ -2080,6 +2080,16 @@ elif(-3 == adderThread.status): error_str = "there was a problem while I was talking to my partner node. Please try again later." self.nodeCommsProblemCount += 1; + if( "Zothar" in self.usersInChan ): + self.privmsg( + "Zothar", + "%s had trouble talking to it's partner node while adding the ref from %s (extended_error_msg is %s)" % (self.botircnick, sender, adderThread.extended_error_msg) + ) + elif( "Zothar_Work" in self.usersInChan ): + self.privmsg( + "Zothar_Work", + "%s had trouble talking to it's partner node while adding the ref from %s (extended_error_msg is %s)" % (self.botircnick, sender, adderThread.extended_error_msg) + ) elif(-4 == adderThread.status): error_str = "the node reports that it already has a peer with that identity. Ref not re-added." elif(-5 == adderThread.status): @@ -2102,7 +2112,7 @@ refs_plural_str = "s" refs_to_go_str = " (%d ref%s to go)" % ( refs_to_go, refs_plural_str ) adderThread.replyfunc("%s%s" % (error_str, refs_to_go_str)) - if( 2 <= self.nodeCommsProblemCount ): + if( 2 <= self.nodeCommsProblemCount ): # Die if we've had too many failures talking to the node self.privmsg( self.channel, @@ -2965,6 +2975,11 @@ except Exception, msg: self.status = -3 self.error_msg = msg + try: + exc_type, exc_value = sys.exc_info()[ :2 ] + self.extended_error_msg = "Exception: type is %s value is %s" % ( exc_type, exc_value ) + except: + self.extended_error_msg = msg if(f != None): f.shutdown(); return From zothar at freenetproject.org Mon Nov 5 23:41:48 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Mon, 5 Nov 2007 23:41:48 +0000 (UTC) Subject: [Pyfreenet] r15697 - trunk/apps/pyFreenet Message-ID: <20071105234148.93981478063@freenetproject.org> Author: zothar Date: 2007-11-05 23:41:48 +0000 (Mon, 05 Nov 2007) New Revision: 15697 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: send seen on channel user list in chunks to avoid overflowing the IRC line length limit Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-05 23:01:05 UTC (rev 15696) +++ trunk/apps/pyFreenet/refbot.py 2007-11-05 23:41:48 UTC (rev 15697) @@ -850,6 +850,8 @@ self.peer_update_interval = 60 self.api_options = [] self.nextWhenTime = 0; + self.nextSeenChannelUsersWhenTime = 0; + self.sendSeenChannelUsersLock = threading.Lock() self.sendRefDirectLock = threading.Lock() if(self.bot2bot_enabled): self.api_options.append( "bot2bot" ); @@ -1566,7 +1568,32 @@ """ Give them our seenChannelUsers """ - self.privmsg( target, "myseenchannelusers %s" % ( self.seenChannelUsers )) + #log( "** sendrefdirect(): sendSeenChannelUsersLock.acquire() before processing target: %s" % ( target )); + self.sendSeenChannelUsersLock.acquire( 1 ); # Lock shared between multiple threads for sending both darknet and opennet refs + # Spread out the chunks of users so we don't trigger the babbler detector of a receiving refbot + nextWhen = 0; + now = long( math.floor( time.time() )); + if( self.nextSeenChannelUsersWhenTime > now): # self.nextSeenChannelUsersWhenTime shared between multiple threads for sending both darknet and opennet refs + nextWhen = self.nextSeenChannelUsersWhenTime - now; + else: + self.nextSeenChannelUsersWhenTime = now; + beginningNextWhenTime = self.nextSeenChannelUsersWhenTime; + #log( "** DEBUG: before: nextWhen: %d nextSeenChannelUsersWhenTime: %d" % ( nextWhen, self.nextSeenChannelUsersWhenTime )); + seenChannelUsers = []; + # Make a copy of the user list + for seenChannelUser in self.seenChannelUsers: + seenChannelUsers.append( seenChannelUser ); + i = 0; + chunkSize = 10; + while( i < len( seenChannelUsers )): + self.after( nextWhen, self.privmsg, target, "myseenchannelusers %s" % ( seenChannelUsers[ i : ( i + chunkSize ) ] )) + i += chunkSize; + delay = random.randint(7,14) # 7-14 seconds between each line + nextWhen += delay; + self.nextSeenChannelUsersWhenTime += delay; + #log( "** DEBUG: after: nextWhen: %d nextWhenTime: %d beginningNextWhenTime: %d diff: %d" % ( nextWhen, self.nextWhenTime, beginningNextWhenTime, self.nextWhenTime - beginningNextWhenTime )); + #log( "** sendrefdirect(): sendSeenChannelUsersLock.release() after processing target: %s" % ( target )); + self.sendSeenChannelUsersLock.release(); # Lock shared between multiple threads for sending both darknet and opennet refs #@-node:sendSeenChannelUsers #@+node:setPeerBotOptions From zothar at freenetproject.org Tue Nov 6 01:04:50 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Tue, 6 Nov 2007 01:04:50 +0000 (UTC) Subject: [Pyfreenet] r15700 - trunk/apps/pyFreenet Message-ID: <20071106010450.60A813905DE@freenetproject.org> Author: zothar Date: 2007-11-06 01:04:49 +0000 (Tue, 06 Nov 2007) New Revision: 15700 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: when popping users off the LIFO because the size is too big, refresh them by pushing them again if they're still in the channel. Add the new nick to the seenChannelUsers LIFO when a user changes their IRC nick. Do seenChannelUses LIFO shrinking when adding users seen by other bots. Send the seen channel users LIFO contents to all bot2bot participating bots, not just announcing ones. Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-06 00:30:54 UTC (rev 15699) +++ trunk/apps/pyFreenet/refbot.py 2007-11-06 01:04:49 UTC (rev 15700) @@ -1200,7 +1200,9 @@ maxSeenChannelUsersCount = len( self.usersInChan ) + 50; log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); while( maxSeenChannelUsersCount < len( self.seenChannelUsers )): - self.seenChannelUsers.pop( 0 ); + oldUser = self.seenChannelUsers.pop( 0 ); + if( oldUser in self.usersInChan ): + self.seenChannelUsers.append( oldUser ); log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); self.seenChannelUsers.append( sender ); log("** DEBUG: self.seenChannelUsers: %s" % ( self.seenChannelUsers )); @@ -1255,6 +1257,17 @@ self.botAnnouncePool[ k ] = target; self.botAnnouncePool.sort(); log("** bots: %s" % ( self.bots.keys() )) + if( not target in self.seenChannelUsers ): + maxSeenChannelUsersCount = len( self.usersInChan ) + 50; + log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); + while( maxSeenChannelUsersCount < len( self.seenChannelUsers )): + oldUser = self.seenChannelUsers.pop( 0 ); + if( oldUser in self.usersInChan ): + self.seenChannelUsers.append( oldUser ); + log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); + self.seenChannelUsers.append( target ); + log("** DEBUG: self.seenChannelUsers: %s" % ( self.seenChannelUsers )); + log("** DEBUG: self.botAnnouncePool: %s" % ( self.botAnnouncePool )); #@-node:post_on_nick #@+node:post_on_part @@ -1339,6 +1352,13 @@ for user in seenChannelUsers: if( not user in self.seenChannelUsers ): self.seenChannelUsers.append( user ); + maxSeenChannelUsersCount = len( self.usersInChan ) + 50; + log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); + while( maxSeenChannelUsersCount < len( self.seenChannelUsers )): + oldUser = self.seenChannelUsers.pop( 0 ); + if( oldUser in self.usersInChan ): + self.seenChannelUsers.append( oldUser ); + log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); #@-node:addSeenChannelUsers #@+node:getPeerUpdate @@ -1609,6 +1629,8 @@ if( botNick not in self.botAnnouncePool ): self.botAnnouncePool.append( botNick ); self.botAnnouncePool.sort(); + if( self.check_bot_peer_has_option( botNick, "bot2bot" )): + if( botNick in self.botAnnouncePool ): self.after( 3, self.sendSeenChannelUsers, botNick ); #@-node:setPeerBotOptions From zothar at freenetproject.org Thu Nov 8 06:01:34 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 06:01:34 +0000 (UTC) Subject: [Pyfreenet] r15702 - trunk/apps/pyFreenet Message-ID: <20071108060134.D2E3747B2FC@freenetproject.org> Author: zothar Date: 2007-11-08 06:01:33 +0000 (Thu, 08 Nov 2007) New Revision: 15702 Modified: trunk/apps/pyFreenet/minibot.py trunk/apps/pyFreenet/refbot.py Log: refbot: improve logging of messages from the IRC server and improve the process of leaving the IRC server in minibot.py Modified: trunk/apps/pyFreenet/minibot.py =================================================================== --- trunk/apps/pyFreenet/minibot.py 2007-11-06 23:52:14 UTC (rev 15701) +++ trunk/apps/pyFreenet/minibot.py 2007-11-08 06:01:33 UTC (rev 15702) @@ -23,7 +23,7 @@ import platform; has_platform_module = True; except: - pass; + pass #@-node:imports #@+node:globals @@ -48,6 +48,11 @@ Terminates the bot """ +class DoneSayingGoodbye(Exception): + """ + Nothing more to say before leaving IRC + """ + class NotReceiving(Exception): """ raised when server puts the bot into la-la land @@ -115,6 +120,8 @@ self.txtimes = [] self.restartCount = 0 self.restartDelay = 45 + self.sock = None + self.stopReason = None #@-node:__init__ #@+node:run @@ -123,6 +130,7 @@ self._keepRunning = True while True: + log("Starting main run() while loop...") self.sched = sched.scheduler(time.time, time.sleep) self._running = True @@ -132,24 +140,37 @@ # suppresses call to self.greetChannel self._restarted = False + self.sock = None self.connect() try: + self._using_scheduler = True self.sched.run() + except DoneSayingGoodbye: + self._keepRunning = False + my_exit( 0 ) + except KeyboardInterrupt: + self._keepRunning = False log("Terminated by user") - self._keepRunning = False - + self.stopReason = "Terminated by user" + except TimeToQuit: self._keepRunning = False + log("Quitting") + self.stopReason = "Quitting" except MemoryError: + traceback.print_exc() log("Got MemoryError exception; Terminating.") self._keepRunning = False + self.stopReason = "MemoryError" except NotReceiving: + self._using_scheduler = False self.sock.close() + self.sock = None self.hasIdentified = False log("** ERROR: server is ignoring us, restarting in 3 seconds...") time.sleep(3) @@ -158,7 +179,10 @@ except: traceback.print_exc() + self._using_scheduler = False + self.part_and_quit( "Uncaught Exception" ) self.sock.close() + self.sock = None self.hasIdentified = False self.restartCount += 1 if( self.restartCount > 7 ): @@ -171,7 +195,14 @@ continue if not self._keepRunning: - self.sock.close() + self._using_scheduler = False + if( None != self.sock ): + if( None != self.stopReason ): + self.part_and_quit( self.stopReason ); + else: + self.part_and_quit( "*Poof*" ) + self.sock.close() + self.sock = None break my_exit( 0 ); @@ -218,17 +249,18 @@ if not connected: log("Couldn't get a connection") + self.sock = None return self._lastRxTime = time.time() # Send the nick to server log("Send nick...") - send('NICK '+ self.nick + "\n") + send('NICK '+ self.nick) # Identify to server log("Sending USER...") - send('USER ' + self.ident + ' ' + self.host + ' bla :' + self.realname + "\n") + send('USER ' + self.ident + ' ' + self.host + ' bla :' + self.realname) # plant initial tasks self.after(0, self._receiver) @@ -242,14 +274,33 @@ #@+others #@+node:on_server_msg - def on_server_msg(self, typ, msg): + def on_server_msg(self, typ, msg, line): """ Handles messages from server """ + svrmsglinefields = line.split(" ", 3) + svrmsg_type = svrmsglinefields[ 1 ] + svrmsg_text = svrmsglinefields[ 3 ] if typ == 'PONG': pass + elif typ in [ '004', '005' ]: + log("** server: %s %s" % (repr(svrmsg_type), svrmsg_text)) + elif typ in [ '001', '002', '003', '250', '251', '255', '265', '266', '372', '375', '376' ]: + log("** server: %s %s" % (repr(typ), msg)) + elif typ in [ '254' ]: + textfields = svrmsg_text.split(":", 1) + svrmsg_int = textfields[ 0 ][ : -1 ] + svrmsg_text = textfields[ 1 ] + log("** server: %s %s %s" % (repr(svrmsg_type), repr(svrmsg_int), svrmsg_text)) + elif typ in [ '366', '401', '403', '433' '442' ]: + textfields = svrmsg_text.split(":", 1) + svrmsg_nick_channel = textfields[ 0 ][ : -1 ] + svrmsg_text = textfields[ 1 ] + log("** server: %s %s %s" % (repr(svrmsg_type), repr(svrmsg_nick_channel), svrmsg_text)) + if( '401' == typ ): + self.die() elif typ not in [ '353', '409' ]: - log("** server: %s %s" % (repr(typ), msg)) + log("** server: %s %s\n**** line=[%s]" % (repr(typ), msg, line)) if "End of /MOTD" in msg: if(not self.hasIdentified): self.after(5, self.identifyPassword) @@ -268,6 +319,15 @@ return #@-node:on_server_msg + #@+node:on_error + def on_error(self, sender, error_type, msg): + + log("** error: %s -> :%s: %s" % (sender, error_type, msg)) + if( "Closing Link" == error_type ): + raise DoneSayingGoodbye + log("?? unhandled error from server: type=%s msg=%s" % ( error_type, msg )) + + #@-node:on_error #@+node:on_notice def on_notice(self, sender, msg): @@ -427,10 +487,17 @@ Called when a raw line comes in from server """ if line.startswith("NOTICE "): - msg = msg = line.split(" ", 1)[-1].strip() + msg = line.split(" ", 1)[-1].strip() self.on_notice("$server", msg) return + if line.startswith("ERROR "): + errorlinefields = line.split(":", 2) + error_type = errorlinefields[ 1 ] + msg = errorlinefields[ 2 ][ 1: ] + self.on_error("$server", error_type, msg) + return + parts = line.split(" ", 3) if( len( parts ) == 0 ): log("?? no parts could be split from: %s" % ( line )); @@ -458,7 +525,7 @@ msg = '' if sender == '$server$': - self.on_server_msg(typ, msg) + self.on_server_msg(typ, msg, line) return if typ == 'JOIN': @@ -498,8 +565,8 @@ quit_msg = '' self.on_quit(sender, quit_msg) else: - log("?? sender=%s typ=%s target=%s msg=%s" % ( - repr(sender), repr(typ), repr(target), repr(msg))) + log("?? sender=%s typ=%s target=%s msg=%s line=[%s]" % ( + repr(sender), repr(typ), repr(target), repr(msg), repr(line))) # NOTE: Should only return as it's been factored out of the above ifs #@-node:on_raw_rx @@ -587,6 +654,36 @@ self.sendline(":%s PRIVMSG %s :%s" % (self.nick, self.channel, line)) #@-node:chanmsg + #@+node:part_and_quit + def part_and_quit(self, reason): + """ + Parts the channel informatively and quits from the IRC server + """ + self.sendline( 'PART ' + self.channel + ' :' + reason ) + self.sendline( 'QUIT :' + reason ) + try: + while( 0 < self.getsendqueuesize() ): + self._receiver(); + self._sender(); + time.sleep( 0.25 ) + startTime = time.time(); + while( 15 > ( time.time() - startTime )): + self._receiver(); + time.sleep( 0.25 ) + except DoneSayingGoodbye: + pass + except NotReceiving: + pass + except socket.error, msg: + log( "** ERROR: socket error: %s" % ( msg )) + except TimeToQuit: + pass + except: + log( "** ERROR: caught unhandled exception while saying goodbye:" ) + traceback.print_exc() + time.sleep( 1 ) + + #@-node:part_and_quit #@+node:pubmsg def pubmsg(self, peernick, *lines): """ @@ -632,7 +729,8 @@ """ if self._running: pri = kw.get('priority', 3) - self.sched.enter(delay, pri, func, args) + return self.sched.enter(delay, pri, func, args) + return None #@-node:after #@+node:die @@ -654,21 +752,28 @@ self.txqueue.append(msg) #@-node:sendline + #@+node:getsendqueuesize + def getsendqueuesize(self): + + return len( self.txqueue ) + + #@-node:getsendqueuesize #@+node:_sender def _sender(self): - fast_send_time = 0.5 - slow_send_time = 2.5 - check_time_range = 30 - slow_send_time_threshold = 6 - next_send_time = fast_send_time - self.txtimes.append(time.time()) - while((time.time() - self.txtimes[ 0 ]) > check_time_range): - self.txtimes = self.txtimes[ 1: ] - recent_sent = len(self.txtimes) - if(recent_sent >= slow_send_time_threshold): - next_send_time = slow_send_time - self.after(next_send_time, self._sender) + if self._using_scheduler: + fast_send_time = 0.5 + slow_send_time = 2.5 + check_time_range = 30 + slow_send_time_threshold = 6 + next_send_time = fast_send_time + self.txtimes.append(time.time()) + while((time.time() - self.txtimes[ 0 ]) > check_time_range): + self.txtimes = self.txtimes[ 1: ] + recent_sent = len(self.txtimes) + if(recent_sent >= slow_send_time_threshold): + next_send_time = slow_send_time + self.after(next_send_time, self._sender) if self.txqueue: msg = self.txqueue.pop(0) @@ -678,7 +783,6 @@ self.sock.send(msg + "\n") - #@-node:_sender #@+node:_receiver def _receiver(self): @@ -696,7 +800,8 @@ else: self.rxbuf.append(c) - self.after(0.1, self._receiver) + if self._using_scheduler: + self.after(0.1, self._receiver) #@-node:_receiver #@+node:_pinger @@ -849,6 +954,14 @@ # action methods #@+others + #@+node:notice + def notice(self, *lines): + """ + Send a notice to peer + """ + self.bot.notice(self.peernick, *lines) + + #@-node:notice #@+node:pubmsg def pubmsg(self, *lines): """ @@ -896,7 +1009,7 @@ if(not is_from_privmsg): self.privmsg("Sorry, but that command will only be honored in a /msg") raise NotPrivateMessage() - self.privmsg("Goodbye, master") + self.notice("Goodbye, master") self.bot.die() #@-node:cmd_die Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-06 23:52:14 UTC (rev 15701) +++ trunk/apps/pyFreenet/refbot.py 2007-11-08 06:01:33 UTC (rev 15702) @@ -1248,14 +1248,14 @@ When another user (or us) have changed nicks (post processing by inheriting class) """ log("** DEBUG: self.usersInChan: %s" % ( self.usersInChan )); + if( sender in self.botAnnouncePool ): + k = self.botAnnouncePool.index( sender ); + self.botAnnouncePool[ k ] = target; + self.botAnnouncePool.sort(); if(self.bots.has_key( sender )): bot_data = self.bots[ sender ] del self.bots[ sender ] self.bots[ target ] = bot_data - if( sender in self.botAnnouncePool ): - k = self.botAnnouncePool.index( sender ); - self.botAnnouncePool[ k ] = target; - self.botAnnouncePool.sort(); log("** bots: %s" % ( self.bots.keys() )) if( not target in self.seenChannelUsers ): maxSeenChannelUsersCount = len( self.usersInChan ) + 50; @@ -1276,9 +1276,9 @@ When another user (or us) have left a channel (post processing by inheriting class) """ log("** DEBUG: self.usersInChan: %s" % ( self.usersInChan )); + if( sender in self.botAnnouncePool ): + self.botAnnouncePool.remove( sender ); if(self.bots.has_key( sender )): - if( sender in self.botAnnouncePool ): - self.botAnnouncePool.remove( sender ); del self.bots[ sender ] log("** bots: %s" % ( self.bots.keys() )) @@ -1289,6 +1289,8 @@ When another user (or us) have quit a server (post processing by inheriting class) """ log("** DEBUG: self.usersInChan: %s" % ( self.usersInChan )); + if( sender in self.botAnnouncePool ): + self.botAnnouncePool.remove( sender ); if(self.bots.has_key( sender )): if( self.bots[ sender ].has_key( "identity" )): identity = self.bots[ sender ][ "identity" ] @@ -1298,8 +1300,6 @@ identity = self.bots[ sender ][ "opennet_identity" ] if( self.botDarknetIdentities.has_key( identity )): del self.botDarknetIdentities[ identity ] - if( sender in self.botAnnouncePool ): - self.botAnnouncePool.remove( sender ); del self.bots[ sender ] log("** bots: %s" % ( self.bots.keys() )) From zothar at freenetproject.org Thu Nov 8 13:40:51 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 13:40:51 +0000 (UTC) Subject: [Pyfreenet] r15704 - trunk/apps/pyFreenet Message-ID: <20071108134051.E2B3D47962C@freenetproject.org> Author: zothar Date: 2007-11-08 13:40:51 +0000 (Thu, 08 Nov 2007) New Revision: 15704 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: indent Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-08 09:30:16 UTC (rev 15703) +++ trunk/apps/pyFreenet/refbot.py 2007-11-08 13:40:51 UTC (rev 15704) @@ -2903,19 +2903,19 @@ def run(self): if( self.peerRef == None ): - try: - openurl = urllib2.urlopen(self.url) - refbuf = openurl.read(20*1024) # read up to 20 KiB - openurl.close() - refmemfile = StringIO.StringIO(refbuf) - reflines = refmemfile.readlines() - refmemfile.close(); - except Exception, msg: - self.status = -2 - self.error_msg = msg - return + try: + openurl = urllib2.urlopen(self.url) + refbuf = openurl.read(20*1024) # read up to 20 KiB + openurl.close() + refmemfile = StringIO.StringIO(refbuf) + reflines = refmemfile.readlines() + refmemfile.close(); + except Exception, msg: + self.status = -2 + self.error_msg = msg + return else: - reflines = self.peerRef + reflines = self.peerRef ref_fieldset = {}; end_found = False for refline in reflines: @@ -2969,87 +2969,87 @@ return try: - f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port ) - if( have_plugin_module ): - try: - self.plugin_args[ "fcpNode" ] = f; - self.plugin_args[ "ref" ] = ref_fieldset; - plugin_result = botplugin.pre_add( self.plugin_args ); - if( plugin_result != None ): - self.status = -6 - self.error_msg = plugin_result + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port ) + if( have_plugin_module ): + try: + self.plugin_args[ "fcpNode" ] = f; + self.plugin_args[ "ref" ] = ref_fieldset; + plugin_result = botplugin.pre_add( self.plugin_args ); + if( plugin_result != None ): + self.status = -6 + self.error_msg = plugin_result + f.shutdown(); + return + except Exception, msg: + log("Got exception calling botplugin.pre_add(): %s" % ( msg )); + returned_peer = f.listpeer( NodeIdentifier = ref_fieldset[ "identity" ] ) + if( type( returned_peer ) == type( [] )): + returned_peer = returned_peer[ 0 ]; + if( returned_peer[ "header" ] == "Peer" ): + self.status = -4 + self.error_msg = "Node already has a peer with that identity" f.shutdown(); return - except Exception, msg: - log("Got exception calling botplugin.pre_add(): %s" % ( msg )); - returned_peer = f.listpeer( NodeIdentifier = ref_fieldset[ "identity" ] ) - if( type( returned_peer ) == type( [] )): - returned_peer = returned_peer[ 0 ]; - if( returned_peer[ "header" ] == "Peer" ): - self.status = -4 - self.error_msg = "Node already has a peer with that identity" - f.shutdown(); - return - if( f.nodeBuild < self.minimumFCPAddNodeBuild ): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.connect((self.tmci_host, self.tmci_port)) - - # wait for something to come in - sock.recv(1) - time.sleep(0.1) - - # wait till node stops sending - while len(select.select([sock], [], [], 0.1)[0]) > 0: - sock.recv(1024) - - sock.send("ADDPEER:\r\n") - for refline in reflines: - refline = refline.strip() - sock.send("%s\r\n" % (refline)) - - # wait for something to come in - sock.recv(1) - time.sleep(0.1) - - # wait till node stops sending - while len(select.select([sock], [], [], 0.1)[0]) > 0: - buf = sock.recv(1024) - sys.stdout.write(buf) - sys.stdout.flush() - print - - sock.close() - else: - addpeer_result = f.addpeer( kwdict = ref_fieldset ) + if( f.nodeBuild < self.minimumFCPAddNodeBuild ): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((self.tmci_host, self.tmci_port)) + + # wait for something to come in + sock.recv(1) + time.sleep(0.1) + + # wait till node stops sending + while len(select.select([sock], [], [], 0.1)[0]) > 0: + sock.recv(1024) + + sock.send("ADDPEER:\r\n") + for refline in reflines: + refline = refline.strip() + sock.send("%s\r\n" % (refline)) + + # wait for something to come in + sock.recv(1) + time.sleep(0.1) + + # wait till node stops sending + while len(select.select([sock], [], [], 0.1)[0]) > 0: + buf = sock.recv(1024) + sys.stdout.write(buf) + sys.stdout.flush() + print + + sock.close() + else: + addpeer_result = f.addpeer( kwdict = ref_fieldset ) except Exception, msg: - self.status = -3 - self.error_msg = msg - try: - exc_type, exc_value = sys.exc_info()[ :2 ] - self.extended_error_msg = "Exception: type is %s value is %s" % ( exc_type, exc_value ) - except: - self.extended_error_msg = msg - if(f != None): - f.shutdown(); - return + self.status = -3 + self.error_msg = msg + try: + exc_type, exc_value = sys.exc_info()[ :2 ] + self.extended_error_msg = "Exception: type is %s value is %s" % ( exc_type, exc_value ) + except: + self.extended_error_msg = msg + if(f != None): + f.shutdown(); + return try: - returned_peer = f.listpeer( NodeIdentifier = ref_fieldset[ "identity" ] ) - if( type( returned_peer ) == type( [] )): - returned_peer = returned_peer[ 0 ]; - if( returned_peer[ "header" ] == "UnknownNodeIdentifier" ): - self.status = -7 - self.error_msg = "Node couldn't add peer for some reason." - f.shutdown(); - return + returned_peer = f.listpeer( NodeIdentifier = ref_fieldset[ "identity" ] ) + if( type( returned_peer ) == type( [] )): + returned_peer = returned_peer[ 0 ]; + if( returned_peer[ "header" ] == "UnknownNodeIdentifier" ): + self.status = -7 + self.error_msg = "Node couldn't add peer for some reason." + f.shutdown(); + return except Exception, msg: - # We'll let this part fail open for now - pass + # We'll let this part fail open for now + pass if( have_plugin_module ): - try: - plugin_result = botplugin.post_add( self.plugin_args ); - except Exception, msg: - log("Got exception calling botplugin.post_add(): %s" % ( msg )); + try: + plugin_result = botplugin.post_add( self.plugin_args ); + except Exception, msg: + log("Got exception calling botplugin.post_add(): %s" % ( msg )); if(self.isDarknetRef): try: if( self.peerRef == None ): @@ -3085,7 +3085,7 @@ f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port ) returned_peer = f.listpeer( NodeIdentifier = self.identity ) if( type( returned_peer ) == type( [] )): - returned_peer = returned_peer[ 0 ]; + returned_peer = returned_peer[ 0 ]; if( returned_peer[ "header" ] == "Peer" ): self.status = 1 self.status_msg = "Node already has a peer with that identity" @@ -3099,7 +3099,7 @@ self.status = -1 self.status_msg = msg if(f != None): - f.shutdown(); + f.shutdown(); return #@-node:class CheckIdentityWithNode @@ -3128,57 +3128,57 @@ #@-node:class GetPeerUpdateHelper #@+node:cidrNetToNumbers def cidrNetToNumbers( networkstr ): - if( networkstr == None ): - return None; - if( networkstr == '' ): - return None; - fields = string.split( networkstr, '/' ); - if( len( fields ) != 2 ): - return None; - return(( fields[ 0 ], int( fields[ 1 ] ))); + if( networkstr == None ): + return None; + if( networkstr == '' ): + return None; + fields = string.split( networkstr, '/' ); + if( len( fields ) != 2 ): + return None; + return(( fields[ 0 ], int( fields[ 1 ] ))); #@-node:cidrNetToNumbers #@+node:getGetHostmaskFromBits def getHostmaskFromBits( bits ): - hostmask = 0; - for i in range( 32 - bits ): - hostmask = hostmask << 1; - hostmask = hostmask + 1; - return refbot_inet_ntoa( hostmask ); + hostmask = 0; + for i in range( 32 - bits ): + hostmask = hostmask << 1; + hostmask = hostmask + 1; + return refbot_inet_ntoa( hostmask ); #@-node:getGetHostmaskFromBits #@+node:getGetNetmaskFromBits def getNetmaskFromBits( bits ): - hostmask = refbot_inet_aton( getHostmaskFromBits( bits )); - fullmask = refbot_inet_aton( "255.255.255.255" ); - netmask = fullmask ^ hostmask; - return refbot_inet_ntoa( netmask ); + hostmask = refbot_inet_aton( getHostmaskFromBits( bits )); + fullmask = refbot_inet_aton( "255.255.255.255" ); + netmask = fullmask ^ hostmask; + return refbot_inet_ntoa( netmask ); #@-node:getGetNetmaskFromBits #@+node:getNetworkAddress def getNetworkAddress( ip, netmask ): - ipdottest = string.split( ip, '.' ); - if( 4 != len( ipdottest )): - return None; - netmaskdottest = string.split( netmask, '.' ); - if( 4 != len( netmaskdottest )): - return None; - ip = refbot_inet_aton( ip ); - netmask = refbot_inet_aton( netmask ); - networkaddr = ip & netmask; - return refbot_inet_ntoa( networkaddr ); + ipdottest = string.split( ip, '.' ); + if( 4 != len( ipdottest )): + return None; + netmaskdottest = string.split( netmask, '.' ); + if( 4 != len( netmaskdottest )): + return None; + ip = refbot_inet_aton( ip ); + netmask = refbot_inet_aton( netmask ); + networkaddr = ip & netmask; + return refbot_inet_ntoa( networkaddr ); #@-node:getNetworkAddress #@+node:getNetworkAddressFromCIDRNet def getNetworkAddressFromCIDRNet( networkstr ): - nums_result = cidrNetToNumbers( networkstr ); - if( None == nums_result ): - return None; - ( network, bits ) = nums_result; - netmask = getNetmaskFromBits( bits ); - if( None == netmask ): - return None; - return getNetworkAddress( network, netmask ); + nums_result = cidrNetToNumbers( networkstr ); + if( None == nums_result ): + return None; + ( network, bits ) = nums_result; + netmask = getNetmaskFromBits( bits ); + if( None == netmask ): + return None; + return getNetworkAddress( network, netmask ); #@+node:getNetworkAddressFromCIDRNet #@+node:getPeerUpdateHelper @@ -3187,26 +3187,26 @@ tpeers = 0 f = None; try: - f = fcp.FCPNode( host = fcp_host, port = fcp_port ) - returned_peerlist = f.listpeers( WithVolatile = True ) + f = fcp.FCPNode( host = fcp_host, port = fcp_port ) + returned_peerlist = f.listpeers( WithVolatile = True ) except Exception, msg: - if(f != None): - f.shutdown() - return { "status" : -1, "status_msg" : msg, "cpeers" : None, "tpeers" : None } + if(f != None): + f.shutdown() + return { "status" : -1, "status_msg" : msg, "cpeers" : None, "tpeers" : None } try: - f.shutdown(); + f.shutdown(); except Exception, msg: - pass # Ignore a failure to end the FCP session as we've got what we want now + pass # Ignore a failure to end the FCP session as we've got what we want now if( type( returned_peerlist ) != type( [] )): - returned_peerlist = [ returned_peerlist ]; + returned_peerlist = [ returned_peerlist ]; for peer in returned_peerlist: - if( peer[ "header" ] != "Peer" ): - break - if( not peer.has_key( "volatile.status" )): - continue; - if( peer[ "volatile.status" ] == "CONNECTED" or peer[ "volatile.status" ] == "BACKED OFF" ): - cpeers += 1 - tpeers += 1 + if( peer[ "header" ] != "Peer" ): + break + if( not peer.has_key( "volatile.status" )): + continue; + if( peer[ "volatile.status" ] == "CONNECTED" or peer[ "volatile.status" ] == "BACKED OFF" ): + cpeers += 1 + tpeers += 1 return { "status" : 0, "status_msg" : "getPeerUpdateHelper completed normally", "cpeers" : cpeers, "tpeers" : tpeers } #@-node:getPeerUpdateHelper From zothar at freenetproject.org Thu Nov 8 13:45:38 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 13:45:38 +0000 (UTC) Subject: [Pyfreenet] r15705 - trunk/apps/pyFreenet Message-ID: <20071108134538.762854797C0@freenetproject.org> Author: zothar Date: 2007-11-08 13:45:38 +0000 (Thu, 08 Nov 2007) New Revision: 15705 Modified: trunk/apps/pyFreenet/minibot.py Log: refbot: try to report the uncaught exception details to Zothar if he's available; fix a minor typo Modified: trunk/apps/pyFreenet/minibot.py =================================================================== --- trunk/apps/pyFreenet/minibot.py 2007-11-08 13:40:51 UTC (rev 15704) +++ trunk/apps/pyFreenet/minibot.py 2007-11-08 13:45:38 UTC (rev 15705) @@ -180,6 +180,30 @@ except: traceback.print_exc() self._using_scheduler = False + # + # + # **FIXME** Try to get a handle on what's causing these problems without broadcasting them to the whole channel + # + try: + exc_type, exc_value = sys.exc_info()[ :2 ] + error_msg = "Exception: type is %s value is %s" % ( exc_type, exc_value ) + except: + error_msg = "Exception while generating the exception message" + if( "Zothar" in self.usersInChan ): + self.privmsg( + "Zothar", + "Uncaught exception in main run() loop of minibot.py: %s" % ( error_msg ) + ) + elif( "Zothar_Work" in self.usersInChan ): + self.privmsg( + "Zothar_Work", + "Uncaught exception in main run() loop of minibot.py: %s" % ( error_msg ) + ) + elif( "Zothar70d_bot" in self.usersInChan ): + self.privmsg( + "Zothar_Work", + "Uncaught exception in main run() loop of minibot.py: %s" % ( error_msg ) + ) self.part_and_quit( "Uncaught Exception" ) self.sock.close() self.sock = None @@ -292,7 +316,7 @@ svrmsg_int = textfields[ 0 ][ : -1 ] svrmsg_text = textfields[ 1 ] log("** server: %s %s %s" % (repr(svrmsg_type), repr(svrmsg_int), svrmsg_text)) - elif typ in [ '366', '401', '403', '433' '442' ]: + elif typ in [ '366', '401', '403', '433', '442' ]: textfields = svrmsg_text.split(":", 1) svrmsg_nick_channel = textfields[ 0 ][ : -1 ] svrmsg_text = textfields[ 1 ] From zothar at freenetproject.org Thu Nov 8 13:57:43 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 13:57:43 +0000 (UTC) Subject: [Pyfreenet] r15706 - trunk/apps/pyFreenet Message-ID: <20071108135743.3639F47B2AE@freenetproject.org> Author: zothar Date: 2007-11-08 13:57:43 +0000 (Thu, 08 Nov 2007) New Revision: 15706 Modified: trunk/apps/pyFreenet/minibot.py Log: refbot: Don't die on all unknown nicks, just when registering with nickserv Modified: trunk/apps/pyFreenet/minibot.py =================================================================== --- trunk/apps/pyFreenet/minibot.py 2007-11-08 13:45:38 UTC (rev 15705) +++ trunk/apps/pyFreenet/minibot.py 2007-11-08 13:57:43 UTC (rev 15706) @@ -321,7 +321,7 @@ svrmsg_nick_channel = textfields[ 0 ][ : -1 ] svrmsg_text = textfields[ 1 ] log("** server: %s %s %s" % (repr(svrmsg_type), repr(svrmsg_nick_channel), svrmsg_text)) - if( '401' == typ ): + if( '401' == typ and "'nickserv'" == svrmsg_nick_channel): self.die() elif typ not in [ '353', '409' ]: log("** server: %s %s\n**** line=[%s]" % (repr(typ), msg, line)) From zothar at freenetproject.org Thu Nov 8 14:45:54 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 14:45:54 +0000 (UTC) Subject: [Pyfreenet] r15707 - trunk/apps/pyFreenet Message-ID: <20071108144554.4E6C947992F@freenetproject.org> Author: zothar Date: 2007-11-08 14:45:54 +0000 (Thu, 08 Nov 2007) New Revision: 15707 Modified: trunk/apps/pyFreenet/minibot.py Log: refbot: improve the exception reporting with tracebacks in minibot.py Modified: trunk/apps/pyFreenet/minibot.py =================================================================== --- trunk/apps/pyFreenet/minibot.py 2007-11-08 13:57:43 UTC (rev 15706) +++ trunk/apps/pyFreenet/minibot.py 2007-11-08 14:45:54 UTC (rev 15707) @@ -178,6 +178,7 @@ continue except: + log("?? ERROR: Uncaught Exception:") traceback.print_exc() self._using_scheduler = False # @@ -186,9 +187,21 @@ # try: exc_type, exc_value = sys.exc_info()[ :2 ] - error_msg = "Exception: type is %s value is %s" % ( exc_type, exc_value ) + traceback_list = traceback.extract_tb( sys.exc_info()[ 2 ] ) + reported_traceback = '' + for traceback_item in traceback_list: + traceback_line = traceback_item[ 1 ] + traceback_file = traceback_item[ 0 ] + traceback_file_fields = traceback_file.split( "/" ) + traceback_file = traceback_file_fields[ -1 ] + reported_traceback += "%d:%s|" % ( traceback_line, traceback_file ) + if( '|' == reported_traceback[ -1 ] ): + reported_traceback = reported_traceback[ : -1 ] + error_msg = "Exception: type=[%s] value=[%s] traceback=[%s]" % ( exc_type, exc_value, reported_traceback ) except: error_msg = "Exception while generating the exception message" + log("?? ERROR: Exception while generating the exception message:") + traceback.print_exc() if( "Zothar" in self.usersInChan ): self.privmsg( "Zothar", From zothar at freenetproject.org Thu Nov 8 15:03:13 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 15:03:13 +0000 (UTC) Subject: [Pyfreenet] r15708 - trunk/apps/pyFreenet Message-ID: <20071108150313.9D28747B768@freenetproject.org> Author: zothar Date: 2007-11-08 15:03:13 +0000 (Thu, 08 Nov 2007) New Revision: 15708 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: hopefully squash the bug reported by ajd47ytkuy3 from his bot Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-08 14:45:54 UTC (rev 15707) +++ trunk/apps/pyFreenet/refbot.py 2007-11-08 15:03:13 UTC (rev 15708) @@ -2132,12 +2132,12 @@ if( "Zothar" in self.usersInChan ): self.privmsg( "Zothar", - "%s had trouble talking to it's partner node while adding the ref from %s (extended_error_msg is %s)" % (self.botircnick, sender, adderThread.extended_error_msg) + "%s had trouble talking to it's partner node while adding the ref from %s (extended_error_msg is %s)" % (self.botircnick, adderThread.sender_irc_nick, adderThread.extended_error_msg) ) elif( "Zothar_Work" in self.usersInChan ): self.privmsg( "Zothar_Work", - "%s had trouble talking to it's partner node while adding the ref from %s (extended_error_msg is %s)" % (self.botircnick, sender, adderThread.extended_error_msg) + "%s had trouble talking to it's partner node while adding the ref from %s (extended_error_msg is %s)" % (self.botircnick, adderThread.sender_irc_nick, adderThread.extended_error_msg) ) elif(-4 == adderThread.status): error_str = "the node reports that it already has a peer with that identity. Ref not re-added." From zothar at freenetproject.org Thu Nov 8 15:12:28 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 15:12:28 +0000 (UTC) Subject: [Pyfreenet] r15709 - trunk/apps/pyFreenet Message-ID: <20071108151228.57FED47987B@freenetproject.org> Author: zothar Date: 2007-11-08 15:12:28 +0000 (Thu, 08 Nov 2007) New Revision: 15709 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: upgrade the addref exception reporting to match that used in minibot.py Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-08 15:03:13 UTC (rev 15708) +++ trunk/apps/pyFreenet/refbot.py 2007-11-08 15:12:28 UTC (rev 15709) @@ -3023,12 +3023,26 @@ addpeer_result = f.addpeer( kwdict = ref_fieldset ) except Exception, msg: self.status = -3 + log("?? ERROR: Uncaught Exception while adding ref:") + traceback.print_exc() self.error_msg = msg try: exc_type, exc_value = sys.exc_info()[ :2 ] - self.extended_error_msg = "Exception: type is %s value is %s" % ( exc_type, exc_value ) + traceback_list = traceback.extract_tb( sys.exc_info()[ 2 ] ) + reported_traceback = '' + for traceback_item in traceback_list: + traceback_line = traceback_item[ 1 ] + traceback_file = traceback_item[ 0 ] + traceback_file_fields = traceback_file.split( "/" ) + traceback_file = traceback_file_fields[ -1 ] + reported_traceback += "%d:%s|" % ( traceback_line, traceback_file ) + if( '|' == reported_traceback[ -1 ] ): + reported_traceback = reported_traceback[ : -1 ] + self.extended_error_msg = "Exception: type=[%s] value=[%s] traceback=[%s]" % ( exc_type, exc_value, reported_traceback ) except: self.extended_error_msg = msg + log("?? ERROR: Exception while generating the exception message:") + traceback.print_exc() if(f != None): f.shutdown(); return From zothar at freenetproject.org Thu Nov 8 16:13:55 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 16:13:55 +0000 (UTC) Subject: [Pyfreenet] r15710 - trunk/apps/pyFreenet Message-ID: <20071108161355.6B91747A371@freenetproject.org> Author: zothar Date: 2007-11-08 16:13:55 +0000 (Thu, 08 Nov 2007) New Revision: 15710 Modified: trunk/apps/pyFreenet/minibot.py trunk/apps/pyFreenet/refbot.py Log: refbot: make FCP log level configurable by config file edit; filepaths are now also ignored on Windows in traceback reports Modified: trunk/apps/pyFreenet/minibot.py =================================================================== --- trunk/apps/pyFreenet/minibot.py 2007-11-08 15:12:28 UTC (rev 15709) +++ trunk/apps/pyFreenet/minibot.py 2007-11-08 16:13:55 UTC (rev 15710) @@ -194,6 +194,8 @@ traceback_file = traceback_item[ 0 ] traceback_file_fields = traceback_file.split( "/" ) traceback_file = traceback_file_fields[ -1 ] + traceback_file_fields = traceback_file.split( "\\" ) + traceback_file = traceback_file_fields[ -1 ] reported_traceback += "%d:%s|" % ( traceback_line, traceback_file ) if( '|' == reported_traceback[ -1 ] ): reported_traceback = reported_traceback[ : -1 ] Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-08 15:12:28 UTC (rev 15709) +++ trunk/apps/pyFreenet/refbot.py 2007-11-08 16:13:55 UTC (rev 15710) @@ -387,13 +387,14 @@ needToSave = True if(opts.has_key('refsperrun')): try: - self.number_of_refs_to_collect = int( opts['refsperrun'] ) + self.number_of_refs_to_collect_configured = int( opts['refsperrun'] ) except: print "Seems you've a bogus value for refsperrun in your config file. Bailing." my_exit( 1 ); else: - self.number_of_refs_to_collect = 10 + self.number_of_refs_to_collect_configured = 10 needToSave = True + self.number_of_refs_to_collect = self.number_of_refs_to_collect_configured if(self.number_of_refs_to_collect <= 0): print "refsperrun is at or below zero. Nothing to do. Quitting." my_exit( 1 ); @@ -418,6 +419,18 @@ else: self.tmci_port = 2323; needToSave = True + if(opts.has_key('fcploglevel')): + try: + self.fcp_log_level = int( opts['fcploglevel'] ) + except: + print "Seems you've a bogus value for fcploglevel in your config file. Bailing." + my_exit( 1 ); + if( 0 > self.fcp_log_level or 7 < self.fcp_log_level ): + print "Seems you've a bogus value for fcploglevel in your config file. The value should be between 0 and 7 inclusive to match the logger verbosity levels defined in fcp/node.py. Bailing." + my_exit( 1 ); + else: + self.fcp_log_level = 3 + needToSave = True # for internal use shadow of MiniBot configs self.irc_host = kw[ 'host' ] @@ -1096,7 +1109,8 @@ f.write(fmt % ("password", repr(self.password))) f.write(fmt % ("greetinterval", repr(self.greet_interval))) f.write(fmt % ("spaminterval", repr(self.spam_interval))) - f.write(fmt % ("refsperrun", repr(self.number_of_refs_to_collect))) + f.write(fmt % ("fcploglevel", repr(self.fcp_log_level))) + f.write(fmt % ("refsperrun", repr(self.number_of_refs_to_collect_configured))) f.write(fmt % ("refs", repr(self.refs))) if(self.bot2bot_configured): f.write(fmt % ("bot2bot", repr('y'))) @@ -3035,6 +3049,8 @@ traceback_file = traceback_item[ 0 ] traceback_file_fields = traceback_file.split( "/" ) traceback_file = traceback_file_fields[ -1 ] + traceback_file_fields = traceback_file.split( "\\" ) + traceback_file = traceback_file_fields[ -1 ] reported_traceback += "%d:%s|" % ( traceback_line, traceback_file ) if( '|' == reported_traceback[ -1 ] ): reported_traceback = reported_traceback[ : -1 ] From zothar at freenetproject.org Thu Nov 8 16:17:58 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 16:17:58 +0000 (UTC) Subject: [Pyfreenet] r15711 - trunk/apps/pyFreenet Message-ID: <20071108161758.DAB5547B381@freenetproject.org> Author: zothar Date: 2007-11-08 16:17:58 +0000 (Thu, 08 Nov 2007) New Revision: 15711 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: Actually use the new fcp log level value :) Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-08 16:13:55 UTC (rev 15710) +++ trunk/apps/pyFreenet/refbot.py 2007-11-08 16:17:58 UTC (rev 15711) @@ -473,7 +473,7 @@ self.nodeDarknetRef = {}; log("Verifying connectivity with node.... (If this hangs, there are problems talking to the node's FCP service)") try: - f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port ) + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, verbosity = self.fcp_log_level ) except Exception, msg: try: f.shutdown() @@ -1910,7 +1910,7 @@ f = None; nodeIsUp = False; try: - f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port ) + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, verbosity = self.fcp_log_level ) nodeIsUp = True; f.shutdown(); self.nodeCommsProblemCount = 0; @@ -2983,7 +2983,7 @@ return try: - f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port ) + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, verbosity = self.fcp_log_level ) if( have_plugin_module ): try: self.plugin_args[ "fcpNode" ] = f; @@ -3112,7 +3112,7 @@ def run(self): try: - f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port ) + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, verbosity = self.fcp_log_level ) returned_peer = f.listpeer( NodeIdentifier = self.identity ) if( type( returned_peer ) == type( [] )): returned_peer = returned_peer[ 0 ]; @@ -3217,7 +3217,7 @@ tpeers = 0 f = None; try: - f = fcp.FCPNode( host = fcp_host, port = fcp_port ) + f = fcp.FCPNode( host = fcp_host, port = fcp_port, verbosity = self.fcp_log_level ) returned_peerlist = f.listpeers( WithVolatile = True ) except Exception, msg: if(f != None): From zothar at freenetproject.org Thu Nov 8 16:31:49 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 16:31:49 +0000 (UTC) Subject: [Pyfreenet] r15712 - trunk/apps/pyFreenet Message-ID: <20071108163149.C70F7390834@freenetproject.org> Author: zothar Date: 2007-11-08 16:31:49 +0000 (Thu, 08 Nov 2007) New Revision: 15712 Modified: trunk/apps/pyFreenet/minibot.py Log: refbot: trivial logging tweak in minibot.py Modified: trunk/apps/pyFreenet/minibot.py =================================================================== --- trunk/apps/pyFreenet/minibot.py 2007-11-08 16:17:58 UTC (rev 15711) +++ trunk/apps/pyFreenet/minibot.py 2007-11-08 16:31:49 UTC (rev 15712) @@ -326,7 +326,7 @@ log("** server: %s %s" % (repr(svrmsg_type), svrmsg_text)) elif typ in [ '001', '002', '003', '250', '251', '255', '265', '266', '372', '375', '376' ]: log("** server: %s %s" % (repr(typ), msg)) - elif typ in [ '254' ]: + elif typ in [ '252', '254' ]: textfields = svrmsg_text.split(":", 1) svrmsg_int = textfields[ 0 ][ : -1 ] svrmsg_text = textfields[ 1 ] From zothar at freenetproject.org Thu Nov 8 16:54:04 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 16:54:04 +0000 (UTC) Subject: [Pyfreenet] r15713 - trunk/apps/pyFreenet Message-ID: <20071108165404.26805479932@freenetproject.org> Author: zothar Date: 2007-11-08 16:54:03 +0000 (Thu, 08 Nov 2007) New Revision: 15713 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: fix the variable scoping issues introduced in r15711 Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-08 16:31:49 UTC (rev 15712) +++ trunk/apps/pyFreenet/refbot.py 2007-11-08 16:54:03 UTC (rev 15713) @@ -831,7 +831,7 @@ log("Getting Peer Update...") temp_cpeers = None; temp_dpeers = None; - peerUpdateCallResult = getPeerUpdateHelper( self.fcp_host, self.fcp_port ) + peerUpdateCallResult = getPeerUpdateHelper( self.fcp_host, self.fcp_port, self.fcp_log_level ) if( peerUpdateCallResult.has_key( "cpeers" )): temp_cpeers = peerUpdateCallResult[ "cpeers" ]; if( peerUpdateCallResult.has_key( "tpeers" )): @@ -1378,7 +1378,7 @@ #@+node:getPeerUpdate def getPeerUpdate(self): - peerUpdaterThread = GetPeerUpdate(self.fcp_host, self.fcp_port) + peerUpdaterThread = GetPeerUpdate(self.fcp_host, self.fcp_port, self.fcp_log_level) self.peerUpdaterThreads.append(peerUpdaterThread) peerUpdaterThread.start() if(self.peer_update_interval > 0): @@ -1699,7 +1699,7 @@ def addref(self, url, replyfunc, sender_irc_nick, peerRef = None, botAddType = None): log("** adding ref: %s" % url) - adderThread = AddRef(self.tmci_host, self.tmci_port, self.fcp_host, self.fcp_port, url, replyfunc, sender_irc_nick, self.irc_host, self.nodeDarknetIdentity, self.nodeDarknetRef, self.hasOpennet, self.nodeOpennetIdentity, self.nodeOpennetRef, self.darknet_trades_enabled, self.opennet_trades_enabled, peerRef, botAddType) + adderThread = AddRef(self.tmci_host, self.tmci_port, self.fcp_host, self.fcp_port, self.fcp_log_level, url, replyfunc, sender_irc_nick, self.irc_host, self.nodeDarknetIdentity, self.nodeDarknetRef, self.hasOpennet, self.nodeOpennetIdentity, self.nodeOpennetRef, self.darknet_trades_enabled, self.opennet_trades_enabled, peerRef, botAddType) self.adderThreads.append(adderThread) adderThread.start() @@ -1730,7 +1730,7 @@ def check_identity_with_node(self, botIdentity): log("** checking identity with node: %s" % ( botIdentity )) - identityCheckerThread = CheckIdentityWithNode(self.fcp_host, self.fcp_port, botIdentity) + identityCheckerThread = CheckIdentityWithNode(self.fcp_host, self.fcp_port, self.fcp_log_level, botIdentity) self.identityCheckerThreads.append(identityCheckerThread) identityCheckerThread.start() @@ -2889,12 +2889,13 @@ minimumFCPAddNodeBuild = 1008; - def __init__(self, tmci_host, tmci_port, fcp_host, fcp_port, url, replyfunc, sender_irc_nick, irc_host, nodeDarknetIdentity, nodeDarknetRef, hasOpennet, nodeOpennetIdentity, nodeOpennetRef, darknet_trades_enabled, opennet_trades_enabled, peerRef, botAddType): + def __init__(self, tmci_host, tmci_port, fcp_host, fcp_port, fcp_log_level, url, replyfunc, sender_irc_nick, irc_host, nodeDarknetIdentity, nodeDarknetRef, hasOpennet, nodeOpennetIdentity, nodeOpennetRef, darknet_trades_enabled, opennet_trades_enabled, peerRef, botAddType): threading.Thread.__init__(self) self.tmci_host = tmci_host self.tmci_port = tmci_port self.fcp_host = fcp_host self.fcp_port = fcp_port + self.fcp_log_level = fcp_log_level self.url = url self.replyfunc = replyfunc self.sender_irc_nick = sender_irc_nick @@ -3102,10 +3103,11 @@ #@-node:class AddRef #@+node:class CheckIdentityWithNode class CheckIdentityWithNode(threading.Thread): - def __init__(self, fcp_host, fcp_port, identity): + def __init__(self, fcp_host, fcp_port, fcp_log_level, identity): threading.Thread.__init__(self) self.fcp_host = fcp_host self.fcp_port = fcp_port + self.fcp_log_level = fcp_log_level self.identity = identity self.status = -1 self.status_msg = None @@ -3135,17 +3137,18 @@ #@-node:class CheckIdentityWithNode #@+node:class GetPeerUpdate class GetPeerUpdate(threading.Thread): - def __init__(self, fcp_host, fcp_port): + def __init__(self, fcp_host, fcp_port, fcp_log_level): threading.Thread.__init__(self) self.fcp_host = fcp_host self.fcp_port = fcp_port + self.fcp_log_level = fcp_log_level self.status = -1 self.status_msg = None self.cpeers = None self.tpeers = None def run(self): - peerUpdateCallResult = getPeerUpdateHelper( self.fcp_host, self.fcp_port ) + peerUpdateCallResult = getPeerUpdateHelper( self.fcp_host, self.fcp_port, self.fcp_log_level ) if( peerUpdateCallResult.has_key( "status" )): self.status = peerUpdateCallResult[ "status" ]; if( peerUpdateCallResult.has_key( "status_msg" )): @@ -3212,12 +3215,12 @@ #@+node:getNetworkAddressFromCIDRNet #@+node:getPeerUpdateHelper -def getPeerUpdateHelper( fcp_host, fcp_port ): +def getPeerUpdateHelper( fcp_host, fcp_port, fcp_log_level ): cpeers = 0 tpeers = 0 f = None; try: - f = fcp.FCPNode( host = fcp_host, port = fcp_port, verbosity = self.fcp_log_level ) + f = fcp.FCPNode( host = fcp_host, port = fcp_port, verbosity = fcp_log_level ) returned_peerlist = f.listpeers( WithVolatile = True ) except Exception, msg: if(f != None): From zothar at freenetproject.org Thu Nov 8 17:45:03 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Thu, 8 Nov 2007 17:45:03 +0000 (UTC) Subject: [Pyfreenet] r15715 - trunk/apps/pyFreenet Message-ID: <20071108174503.E07A2479862@freenetproject.org> Author: zothar Date: 2007-11-08 17:45:03 +0000 (Thu, 08 Nov 2007) New Revision: 15715 Modified: trunk/apps/pyFreenet/minibot.py trunk/apps/pyFreenet/refbot.py Log: refbot: log the easy stuff (users of log()) to a file, configured by manually editing the config file, no file logging by default Modified: trunk/apps/pyFreenet/minibot.py =================================================================== --- trunk/apps/pyFreenet/minibot.py 2007-11-08 17:13:21 UTC (rev 15714) +++ trunk/apps/pyFreenet/minibot.py 2007-11-08 17:45:03 UTC (rev 15715) @@ -30,6 +30,8 @@ progname = sys.argv[0] args = sys.argv[1:] nargs = len(args) +logfilepath = None +logfile = None #@-node:globals #@+node:exceptions @@ -90,6 +92,8 @@ - peerclass - class to be used for encapsulating conversations with peer users - default PeerConversation """ + global logfilepath, logfile + self.host = kw['host'] self.port = kw.get('port', 6667) self.channel = kw['channel'] @@ -98,6 +102,8 @@ self.ident = kw.get('ident', 'FreenetRefBot') self.password = kw['password'] self.peerclass = kw.get('peerclass', PrivateChat) + logfilepath = kw.get('logfilepath', None) + logfile = None realname = kw.get('realname', None) if not realname: @@ -272,7 +278,7 @@ # sock.settimeout(15) #except: # # Ignore on pre-2.3 hopefully - # print "Setting default socket timeout failed; running in Python pre-2.3?"; + # log("Setting default socket timeout failed; running in Python pre-2.3?") # traceback.print_exc() for ip in ip_addresses: log("Connect to %s:%s" % (ip, port)) @@ -376,7 +382,7 @@ try: waitseconds = int( waitsecondsbuf ) except: - print "Failed to parse '%s' as an integer. Perhaps minibot.py needs an update for an IRC server behavior change." % ( waitsecondsbuf ) + log("?? ERROR: Failed to parse '%s' as an integer. Perhaps minibot.py needs an update for an IRC server behavior change." % ( waitsecondsbuf )) my_exit( 1 ); log("Just registered password, waiting %d seconds..." % ( waitseconds )) self.after(waitseconds + 1, self.registerPassword) @@ -1100,8 +1106,15 @@ #@-node:class PrivateChat #@+node:log def log(msg): - sys.stdout.write("%s: %s\n" % (time.strftime("%Y%m%d-%H%M%S"), msg)) + global logfilepath, logfile + + formatted_msg = "%s: %s\n" % (time.strftime("%Y%m%d-%H%M%S"), msg) + sys.stdout.write(formatted_msg) sys.stdout.flush() + if( None == logfile and None != logfilepath ): + logfile = file( logfilepath, "w+" ) + if( None != logfile ): + logfile.write(formatted_msg) #@-node:log #@+node:main @@ -1118,10 +1131,14 @@ #@-node:main #@+node:my_exit def my_exit( exit_status ): - log("Closing program with an exit status of %s" % ( exit_status )) - if( not has_platform_module or ( has_platform_module and "Windows" == platform.system())): - raw_input( "Hit Enter or Return to continue..." ); - sys.exit( exit_status ); + global logfilepath, logfile + + log("Closing program with an exit status of %s" % ( exit_status )) + if( None != logfile ): + logfile.close() + if( not has_platform_module or ( has_platform_module and "Windows" == platform.system())): + raw_input( "Hit Enter or Return to continue..." ); + sys.exit( exit_status ); #@+node:my_exit #@+node:mainline Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-08 17:13:21 UTC (rev 15714) +++ trunk/apps/pyFreenet/refbot.py 2007-11-08 17:45:03 UTC (rev 15715) @@ -431,6 +431,14 @@ else: self.fcp_log_level = 3 needToSave = True + if(opts.has_key('logfilepath')): + self.log_file_path = opts['logfilepath'] + if( "None" == self.log_file_path ): + self.log_file_path = None + else: + self.log_file_path = None + needToSave = True + kw['logfilepath'] = self.log_file_path # for internal use shadow of MiniBot configs self.irc_host = kw[ 'host' ] @@ -1110,6 +1118,7 @@ f.write(fmt % ("greetinterval", repr(self.greet_interval))) f.write(fmt % ("spaminterval", repr(self.spam_interval))) f.write(fmt % ("fcploglevel", repr(self.fcp_log_level))) + f.write(fmt % ("logfilepath", repr(self.log_file_path))) f.write(fmt % ("refsperrun", repr(self.number_of_refs_to_collect_configured))) f.write(fmt % ("refs", repr(self.refs))) if(self.bot2bot_configured): From zothar at freenetproject.org Fri Nov 9 15:30:28 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Fri, 9 Nov 2007 15:30:28 +0000 (UTC) Subject: [Pyfreenet] r15719 - trunk/apps/pyFreenet Message-ID: <20071109153028.35C1047A1DB@freenetproject.org> Author: zothar Date: 2007-11-09 15:30:28 +0000 (Fri, 09 Nov 2007) New Revision: 15719 Modified: trunk/apps/pyFreenet/bogon-bn-agg.txt Log: Update bogon-bn-agg.txt Modified: trunk/apps/pyFreenet/bogon-bn-agg.txt =================================================================== --- trunk/apps/pyFreenet/bogon-bn-agg.txt 2007-11-09 13:56:59 UTC (rev 15718) +++ trunk/apps/pyFreenet/bogon-bn-agg.txt 2007-11-09 15:30:28 UTC (rev 15719) @@ -11,17 +11,16 @@ 46.0.0.0/8 49.0.0.0/8 50.0.0.0/8 -94.0.0.0/7 100.0.0.0/6 104.0.0.0/5 -112.0.0.0/6 +112.0.0.0/7 127.0.0.0/8 169.254.0.0/16 172.16.0.0/12 173.0.0.0/8 174.0.0.0/7 176.0.0.0/5 -184.0.0.0/6 +184.0.0.0/7 192.0.2.0/24 192.168.0.0/16 197.0.0.0/8 From zothar at freenetproject.org Fri Nov 9 16:46:37 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Fri, 9 Nov 2007 16:46:37 +0000 (UTC) Subject: [Pyfreenet] r15720 - trunk/apps/pyFreenet/fcp Message-ID: <20071109164637.5FD2B47960A@freenetproject.org> Author: zothar Date: 2007-11-09 16:46:37 +0000 (Fri, 09 Nov 2007) New Revision: 15720 Modified: trunk/apps/pyFreenet/fcp/node.py Log: pyFreenet: Add support for logging via a function using the logfunc argument on FCPNode instantiation Modified: trunk/apps/pyFreenet/fcp/node.py =================================================================== --- trunk/apps/pyFreenet/fcp/node.py 2007-11-09 15:30:28 UTC (rev 15719) +++ trunk/apps/pyFreenet/fcp/node.py 2007-11-09 16:46:37 UTC (rev 15720) @@ -224,7 +224,9 @@ - port - port number, defaults to environment variable FCP_PORT, and if this doesn't exist, then defaultFCPPort - logfile - a pathname or writable file object, to which log messages - should be written, defaults to stdout + should be written, defaults to stdout unless logfunc is specified + - logfunc - a function to which log messages should be written or None + for no such function should be used, defaults to None - verbosity - how detailed the log messages should be, defaults to 0 (silence) - socketTimeout - value to pass to socket object's settimeout() if @@ -257,13 +259,17 @@ self.socketTimeout = kw.get('socketTimeout', None) # set up the logger - logfile = kw.get('logfile', None) or sys.stdout - if not hasattr(logfile, 'write'): + logfile = kw.get('logfile', None) + logfunc = kw.get('logfunc', None) + if(None == logfile and None == logfunc): + logfile = sys.stdout + if(None != logfile and not hasattr(logfile, 'write')): # might be a pathname if not isinstance(logfile, str): raise Exception("Bad logfile '%s', must be pathname or file object" % logfile) logfile = file(logfile, "a") self.logfile = logfile + self.logfunc = logfunc self.verbosity = kw.get('verbosity', defaultVerbosity) # try to connect to node @@ -1851,7 +1857,7 @@ del self.socket # and close the logfile - if self.logfile not in [sys.stdout, sys.stderr]: + if None != self.logfile and self.logfile not in [sys.stdout, sys.stderr]: self.logfile.close() log(DETAIL, "shutdown: done?") @@ -2582,11 +2588,18 @@ if level > self.verbosity: return - if not msg.endswith("\n"): msg += "\n" + if(None != self.logfile): + if not msg.endswith("\n"): + msg += "\n" + self.logfile.write(msg) + self.logfile.flush() + if(None != self.logfunc): + while( msg.endswith("\n") ): + msg = msg[ : -1 ] + msglines = msg.split("\n") + for msgline in msglines: + self.logfunc(msgline) - self.logfile.write(msg) - self.logfile.flush() - #@-node:_log #@-others #@-node:Low Level Methods @@ -2850,8 +2863,8 @@ if not msg.endswith("\n"): msg += "\n" - self.logfile.write(msg) - self.logfile.flush() + sys.stdout.write(msg) + sys.stdout.flush() #@-node:defaultLogger #@-others From zothar at freenetproject.org Fri Nov 9 16:49:42 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Fri, 9 Nov 2007 16:49:42 +0000 (UTC) Subject: [Pyfreenet] r15721 - trunk/apps/pyFreenet Message-ID: <20071109164942.F3CB947992A@freenetproject.org> Author: zothar Date: 2007-11-09 16:49:42 +0000 (Fri, 09 Nov 2007) New Revision: 15721 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: Use FCPNode's new logfunc capability and bump the minimum dependency revisions Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-09 16:46:37 UTC (rev 15720) +++ trunk/apps/pyFreenet/refbot.py 2007-11-09 16:49:42 UTC (rev 15721) @@ -71,8 +71,8 @@ bot_install_directory = '.'; if( '/' != bot_install_directory[ -1 ] ): bot_install_directory += '/'; - minimumFCPNodeRevision = 14145; - minimumMiniBotRevision = 11957; + minimumFCPNodeRevision = 15720; + minimumMiniBotRevision = 15715; minimumNodeBuild = 1045; svnLongRevision = "$Revision$" svnRevision = svnLongRevision[ 11 : -2 ] @@ -481,7 +481,7 @@ self.nodeDarknetRef = {}; log("Verifying connectivity with node.... (If this hangs, there are problems talking to the node's FCP service)") try: - f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, verbosity = self.fcp_log_level ) + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, logfunc = log, verbosity = self.fcp_log_level ) except Exception, msg: try: f.shutdown() @@ -1919,7 +1919,7 @@ f = None; nodeIsUp = False; try: - f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, verbosity = self.fcp_log_level ) + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, logfunc = log, verbosity = self.fcp_log_level ) nodeIsUp = True; f.shutdown(); self.nodeCommsProblemCount = 0; @@ -2993,7 +2993,7 @@ return try: - f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, verbosity = self.fcp_log_level ) + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, logfunc = log, verbosity = self.fcp_log_level ) if( have_plugin_module ): try: self.plugin_args[ "fcpNode" ] = f; @@ -3123,7 +3123,7 @@ def run(self): try: - f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, verbosity = self.fcp_log_level ) + f = fcp.FCPNode( host = self.fcp_host, port = self.fcp_port, logfunc = log, verbosity = self.fcp_log_level ) returned_peer = f.listpeer( NodeIdentifier = self.identity ) if( type( returned_peer ) == type( [] )): returned_peer = returned_peer[ 0 ]; @@ -3229,7 +3229,7 @@ tpeers = 0 f = None; try: - f = fcp.FCPNode( host = fcp_host, port = fcp_port, verbosity = fcp_log_level ) + f = fcp.FCPNode( host = fcp_host, port = fcp_port, logfunc = log, verbosity = fcp_log_level ) returned_peerlist = f.listpeers( WithVolatile = True ) except Exception, msg: if(f != None): From zothar at freenetproject.org Fri Nov 9 17:12:45 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Fri, 9 Nov 2007 17:12:45 +0000 (UTC) Subject: [Pyfreenet] r15722 - trunk/apps/pyFreenet Message-ID: <20071109171245.1E0A84790B0@freenetproject.org> Author: zothar Date: 2007-11-09 17:12:44 +0000 (Fri, 09 Nov 2007) New Revision: 15722 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: Provide for a separate fcp log level for peer updates Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-09 16:49:42 UTC (rev 15721) +++ trunk/apps/pyFreenet/refbot.py 2007-11-09 17:12:44 UTC (rev 15722) @@ -431,6 +431,18 @@ else: self.fcp_log_level = 3 needToSave = True + if(opts.has_key('fcploglevel.peerupdate')): + try: + self.peer_update_fcp_log_level = int( opts['fcploglevel.peerupdate'] ) + except: + print "Seems you've a bogus value for fcploglevel.peerupdate in your config file. Bailing." + my_exit( 1 ); + if( 0 > self.peer_update_fcp_log_level or 7 < self.peer_update_fcp_log_level ): + print "Seems you've a bogus value for fcploglevel.peerupdate in your config file. The value should be between 0 and 7 inclusive to match the logger verbosity levels defined in fcp/node.py. Bailing." + my_exit( 1 ); + else: + self.peer_update_fcp_log_level = 3 + needToSave = True if(opts.has_key('logfilepath')): self.log_file_path = opts['logfilepath'] if( "None" == self.log_file_path ): @@ -839,7 +851,7 @@ log("Getting Peer Update...") temp_cpeers = None; temp_dpeers = None; - peerUpdateCallResult = getPeerUpdateHelper( self.fcp_host, self.fcp_port, self.fcp_log_level ) + peerUpdateCallResult = getPeerUpdateHelper( self.fcp_host, self.fcp_port, self.peer_update_fcp_log_level ) if( peerUpdateCallResult.has_key( "cpeers" )): temp_cpeers = peerUpdateCallResult[ "cpeers" ]; if( peerUpdateCallResult.has_key( "tpeers" )): @@ -1118,6 +1130,7 @@ f.write(fmt % ("greetinterval", repr(self.greet_interval))) f.write(fmt % ("spaminterval", repr(self.spam_interval))) f.write(fmt % ("fcploglevel", repr(self.fcp_log_level))) + f.write(fmt % ("fcploglevel.peerupdate", repr(self.peer_update_fcp_log_level))) f.write(fmt % ("logfilepath", repr(self.log_file_path))) f.write(fmt % ("refsperrun", repr(self.number_of_refs_to_collect_configured))) f.write(fmt % ("refs", repr(self.refs))) @@ -1387,7 +1400,7 @@ #@+node:getPeerUpdate def getPeerUpdate(self): - peerUpdaterThread = GetPeerUpdate(self.fcp_host, self.fcp_port, self.fcp_log_level) + peerUpdaterThread = GetPeerUpdate(self.fcp_host, self.fcp_port, self.peer_update_fcp_log_level) self.peerUpdaterThreads.append(peerUpdaterThread) peerUpdaterThread.start() if(self.peer_update_interval > 0): From zothar at freenetproject.org Fri Nov 9 17:47:29 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Fri, 9 Nov 2007 17:47:29 +0000 (UTC) Subject: [Pyfreenet] r15726 - trunk/apps/pyFreenet Message-ID: <20071109174729.299FF47A1DB@freenetproject.org> Author: zothar Date: 2007-11-09 17:47:28 +0000 (Fri, 09 Nov 2007) New Revision: 15726 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: Now track either three times the number of users in the channel or the number of users in the channel plus 100, whichever is more. Fix the config file variable name for the peerupdate fcp log level Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-09 17:46:40 UTC (rev 15725) +++ trunk/apps/pyFreenet/refbot.py 2007-11-09 17:47:28 UTC (rev 15726) @@ -431,14 +431,14 @@ else: self.fcp_log_level = 3 needToSave = True - if(opts.has_key('fcploglevel.peerupdate')): + if(opts.has_key('fcploglevel_peerupdate')): try: - self.peer_update_fcp_log_level = int( opts['fcploglevel.peerupdate'] ) + self.peer_update_fcp_log_level = int( opts['fcploglevel_peerupdate'] ) except: - print "Seems you've a bogus value for fcploglevel.peerupdate in your config file. Bailing." + print "Seems you've a bogus value for fcploglevel_peerupdate in your config file. Bailing." my_exit( 1 ); if( 0 > self.peer_update_fcp_log_level or 7 < self.peer_update_fcp_log_level ): - print "Seems you've a bogus value for fcploglevel.peerupdate in your config file. The value should be between 0 and 7 inclusive to match the logger verbosity levels defined in fcp/node.py. Bailing." + print "Seems you've a bogus value for fcploglevel_peerupdate in your config file. The value should be between 0 and 7 inclusive to match the logger verbosity levels defined in fcp/node.py. Bailing." my_exit( 1 ); else: self.peer_update_fcp_log_level = 3 @@ -1130,7 +1130,7 @@ f.write(fmt % ("greetinterval", repr(self.greet_interval))) f.write(fmt % ("spaminterval", repr(self.spam_interval))) f.write(fmt % ("fcploglevel", repr(self.fcp_log_level))) - f.write(fmt % ("fcploglevel.peerupdate", repr(self.peer_update_fcp_log_level))) + f.write(fmt % ("fcploglevel_peerupdate", repr(self.peer_update_fcp_log_level))) f.write(fmt % ("logfilepath", repr(self.log_file_path))) f.write(fmt % ("refsperrun", repr(self.number_of_refs_to_collect_configured))) f.write(fmt % ("refs", repr(self.refs))) @@ -1233,8 +1233,8 @@ log("** DEBUG: post_on_join() called with sender: %s target: %s" % ( sender, target )); log("** DEBUG: self.usersInChan: %s" % ( self.usersInChan )); if( not sender in self.seenChannelUsers ): - maxSeenChannelUsersCount = len( self.usersInChan ) + 50; - log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); + maxSeenChannelUsersCount = getMaxSeenChannelUsers( len( self.usersInChan )); + #log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); while( maxSeenChannelUsersCount < len( self.seenChannelUsers )): oldUser = self.seenChannelUsers.pop( 0 ); if( oldUser in self.usersInChan ): @@ -1294,7 +1294,7 @@ self.bots[ target ] = bot_data log("** bots: %s" % ( self.bots.keys() )) if( not target in self.seenChannelUsers ): - maxSeenChannelUsersCount = len( self.usersInChan ) + 50; + maxSeenChannelUsersCount = getMaxSeenChannelUsers( len( self.usersInChan )); log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); while( maxSeenChannelUsersCount < len( self.seenChannelUsers )): oldUser = self.seenChannelUsers.pop( 0 ); @@ -1388,7 +1388,7 @@ for user in seenChannelUsers: if( not user in self.seenChannelUsers ): self.seenChannelUsers.append( user ); - maxSeenChannelUsersCount = len( self.usersInChan ) + 50; + maxSeenChannelUsersCount = getMaxSeenChannelUsers( len( self.usersInChan )); log("** DEBUG: maxSeenChannelUsersCount: %s len( self.seenChannelUsers ): %s" % ( maxSeenChannelUsersCount, len( self.seenChannelUsers ))); while( maxSeenChannelUsersCount < len( self.seenChannelUsers )): oldUser = self.seenChannelUsers.pop( 0 ); @@ -3310,6 +3310,13 @@ bot.run() #@-node:main +#@+node:getMaxSeenChannelUsers +def getMaxSeenChannelUsers( usersInChannelCount ): + temp1 = usersInChannelCount * 3 + temp1 = usersInChannelCount + 100 + return max( temp1, temp2 ) + +#@-node:getMaxSeenChannelUsers #@+node:readBogonFile def readBogonFile( bogon_filename, bogon_list_adder_callback ): bogon_file = open( bogon_filename ); From zothar at freenetproject.org Fri Nov 9 17:48:30 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Fri, 9 Nov 2007 17:48:30 +0000 (UTC) Subject: [Pyfreenet] r15727 - trunk/apps/pyFreenet Message-ID: <20071109174830.29C9F47A2D2@freenetproject.org> Author: zothar Date: 2007-11-09 17:48:29 +0000 (Fri, 09 Nov 2007) New Revision: 15727 Modified: trunk/apps/pyFreenet/refbot.py Log: refbot: Fix a logic error in r15726 Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-09 17:47:28 UTC (rev 15726) +++ trunk/apps/pyFreenet/refbot.py 2007-11-09 17:48:29 UTC (rev 15727) @@ -3313,7 +3313,7 @@ #@+node:getMaxSeenChannelUsers def getMaxSeenChannelUsers( usersInChannelCount ): temp1 = usersInChannelCount * 3 - temp1 = usersInChannelCount + 100 + temp2 = usersInChannelCount + 100 return max( temp1, temp2 ) #@-node:getMaxSeenChannelUsers From zothar at freenetproject.org Sun Nov 11 03:42:16 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Sun, 11 Nov 2007 03:42:16 +0000 (UTC) Subject: [Pyfreenet] r15754 - trunk/apps/pyFreenet Message-ID: <20071111034216.E3A4D4793D2@freenetproject.org> Author: zothar Date: 2007-11-11 03:42:16 +0000 (Sun, 11 Nov 2007) New Revision: 15754 Modified: trunk/apps/pyFreenet/minibot.py trunk/apps/pyFreenet/refbot.py Log: refbot: log tracebacks through our standard logging target facilities Modified: trunk/apps/pyFreenet/minibot.py =================================================================== --- trunk/apps/pyFreenet/minibot.py 2007-11-10 22:59:03 UTC (rev 15753) +++ trunk/apps/pyFreenet/minibot.py 2007-11-11 03:42:16 UTC (rev 15754) @@ -168,7 +168,7 @@ self.stopReason = "Quitting" except MemoryError: - traceback.print_exc() + log_traceback() log("Got MemoryError exception; Terminating.") self._keepRunning = False self.stopReason = "MemoryError" @@ -185,7 +185,7 @@ except: log("?? ERROR: Uncaught Exception:") - traceback.print_exc() + log_traceback() self._using_scheduler = False # # @@ -209,7 +209,7 @@ except: error_msg = "Exception while generating the exception message" log("?? ERROR: Exception while generating the exception message:") - traceback.print_exc() + log_traceback() if( "Zothar" in self.usersInChan ): self.privmsg( "Zothar", @@ -279,7 +279,7 @@ #except: # # Ignore on pre-2.3 hopefully # log("Setting default socket timeout failed; running in Python pre-2.3?") - # traceback.print_exc() + # log_traceback() for ip in ip_addresses: log("Connect to %s:%s" % (ip, port)) try: @@ -289,7 +289,7 @@ break except: #sock.close() - #traceback.print_exc() + #log_traceback() log("Failed to connect to %s:%s" % (ip, port)) if not connected: @@ -725,7 +725,7 @@ pass except: log( "** ERROR: caught unhandled exception while saying goodbye:" ) - traceback.print_exc() + log_traceback() time.sleep( 1 ) #@-node:part_and_quit @@ -1117,6 +1117,17 @@ logfile.write(formatted_msg) #@-node:log +#@+node:log_traceback +def log_traceback(): + traceback_items = traceback.format_exception( sys.exc_info()[ 0 ], sys.exc_info()[ 1 ], sys.exc_info()[ 2 ] ) + for traceback_item in traceback_items: + while( traceback_item.endswith("\n") ): + traceback_item = traceback_item[ : -1 ] + traceback_item_lines = traceback_item.split("\n") + for traceback_item_line in traceback_item_lines: + log(traceback_item_line) + +#@-node:log_traceback #@+node:main def main(): Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-10 22:59:03 UTC (rev 15753) +++ trunk/apps/pyFreenet/refbot.py 2007-11-11 03:42:16 UTC (rev 15754) @@ -25,7 +25,7 @@ import urlparse import fcp -from minibot import log, MiniBot, PrivateChat, my_exit +from minibot import log, log_traceback, MiniBot, PrivateChat, my_exit have_plugin_module = False; try: @@ -3061,7 +3061,7 @@ except Exception, msg: self.status = -3 log("?? ERROR: Uncaught Exception while adding ref:") - traceback.print_exc() + log_traceback() self.error_msg = msg try: exc_type, exc_value = sys.exc_info()[ :2 ] @@ -3081,7 +3081,7 @@ except: self.extended_error_msg = msg log("?? ERROR: Exception while generating the exception message:") - traceback.print_exc() + log_traceback() if(f != None): f.shutdown(); return From zothar at freenetproject.org Sun Nov 11 15:11:59 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Sun, 11 Nov 2007 15:11:59 +0000 (UTC) Subject: [Pyfreenet] r15759 - trunk/apps/pyFreenet Message-ID: <20071111151159.D5543479629@freenetproject.org> Author: zothar Date: 2007-11-11 15:11:59 +0000 (Sun, 11 Nov 2007) New Revision: 15759 Modified: trunk/apps/pyFreenet/minibot.py trunk/apps/pyFreenet/refbot.py Log: refbot: combine all multi-line bot-sent messages under a single lock and delay tracking variable Modified: trunk/apps/pyFreenet/minibot.py =================================================================== --- trunk/apps/pyFreenet/minibot.py 2007-11-11 15:07:03 UTC (rev 15758) +++ trunk/apps/pyFreenet/minibot.py 2007-11-11 15:11:59 UTC (rev 15759) @@ -117,7 +117,6 @@ self._running = False self.peers = {} self.lastSendTime = time.time() - self.sendlock = threading.Lock() self.usersInChan = [] Modified: trunk/apps/pyFreenet/refbot.py =================================================================== --- trunk/apps/pyFreenet/refbot.py 2007-11-11 15:07:03 UTC (rev 15758) +++ trunk/apps/pyFreenet/refbot.py 2007-11-11 15:11:59 UTC (rev 15759) @@ -873,7 +873,8 @@ self.haveSentDownloadLink = False self.lastSendTime = time.time() - self.sendlock = threading.Lock() + self.multilineSendLock = threading.Lock() + self.nextMultilineSendWhenTime = 0; self.adderThreads = [] self.directOrDieLock = threading.Lock() @@ -882,10 +883,6 @@ self.peerUpdaterThreads = [] self.peer_update_interval = 60 self.api_options = [] - self.nextWhenTime = 0; - self.nextSeenChannelUsersWhenTime = 0; - self.sendSeenChannelUsersLock = threading.Lock() - self.sendRefDirectLock = threading.Lock() if(self.bot2bot_enabled): self.api_options.append( "bot2bot" ); if(self.bot2bot_announces_enabled): @@ -1624,17 +1621,17 @@ """ Give them our seenChannelUsers """ - #log( "** sendrefdirect(): sendSeenChannelUsersLock.acquire() before processing target: %s" % ( target )); - self.sendSeenChannelUsersLock.acquire( 1 ); # Lock shared between multiple threads for sending both darknet and opennet refs + #log( "** sendrefdirect(): multilineSendLock.acquire() before processing target: %s" % ( target )); + self.multilineSendLock.acquire( 1 ); # Lock shared between all multiline senders # Spread out the chunks of users so we don't trigger the babbler detector of a receiving refbot nextWhen = 0; now = long( math.floor( time.time() )); - if( self.nextSeenChannelUsersWhenTime > now): # self.nextSeenChannelUsersWhenTime shared between multiple threads for sending both darknet and opennet refs - nextWhen = self.nextSeenChannelUsersWhenTime - now; + if( self.nextMultilineSendWhenTime > now): # self.nextMultilineSendWhenTime shared between all multiline senders + nextWhen = self.nextMultilineSendWhenTime - now; else: - self.nextSeenChannelUsersWhenTime = now; - beginningNextWhenTime = self.nextSeenChannelUsersWhenTime; - #log( "** DEBUG: before: nextWhen: %d nextSeenChannelUsersWhenTime: %d" % ( nextWhen, self.nextSeenChannelUsersWhenTime )); + self.nextMultilineSendWhenTime = now; + beginningNextWhenTime = self.nextMultilineSendWhenTime; + #log( "** DEBUG: before: nextWhen: %d nextMultilineSendWhenTime: %d" % ( nextWhen, self.nextMultilineSendWhenTime )); seenChannelUsers = []; # Make a copy of the user list for seenChannelUser in self.seenChannelUsers: @@ -1646,10 +1643,10 @@ i += chunkSize; delay = random.randint(7,14) # 7-14 seconds between each line nextWhen += delay; - self.nextSeenChannelUsersWhenTime += delay; - #log( "** DEBUG: after: nextWhen: %d nextWhenTime: %d beginningNextWhenTime: %d diff: %d" % ( nextWhen, self.nextWhenTime, beginningNextWhenTime, self.nextWhenTime - beginningNextWhenTime )); - #log( "** sendrefdirect(): sendSeenChannelUsersLock.release() after processing target: %s" % ( target )); - self.sendSeenChannelUsersLock.release(); # Lock shared between multiple threads for sending both darknet and opennet refs + self.nextMultilineSendWhenTime += delay; + #log( "** DEBUG: after: nextWhen: %d nextMultilineSendWhenTime: %d beginningNextWhenTime: %d diff: %d" % ( nextWhen, self.nextMultilineSendWhenTime, beginningNextWhenTime, self.nextMultilineSendWhenTime - beginningNextWhenTime )); + #log( "** sendrefdirect(): multilineSendLock.release() after processing target: %s" % ( target )); + self.multilineSendLock.release(); # Lock shared between all multiline senders #@-node:sendSeenChannelUsers #@+node:setPeerBotOptions @@ -1995,26 +1992,26 @@ nodeOpennetRefKeys = self.nodeOpennetRef.keys() nodeOpennetRefKeys.sort() - #log( "** sendrefdirect(): sendRefDirectLock.acquire() before processing peernick: %s" % ( peernick )); - self.sendRefDirectLock.acquire( 1 ); # Lock shared between multiple threads for sending both darknet and opennet refs + #log( "** sendrefdirect(): multilineSendLock.acquire() before processing peernick: %s" % ( peernick )); + self.multilineSendLock.acquire( 1 ); # Lock shared between all multiline senders # Spread out the lines of the ref so we don't trigger the babbler detector of a receiving refbot nextWhen = 0; now = long( math.floor( time.time() )); - if( self.nextWhenTime > now): # self.nextWhenTime shared between multiple threads for sending both darknet and opennet refs - nextWhen = self.nextWhenTime - now; + if( self.nextMultilineSendWhenTime > now): # self.nextMultilineSendWhenTime shared between all multiline senders + nextWhen = self.nextMultilineSendWhenTime - now; else: - self.nextWhenTime = now; - beginningNextWhenTime = self.nextWhenTime; - #log( "** DEBUG: before: nextWhen: %d nextWhenTime: %d" % ( nextWhen, self.nextWhenTime )); + self.nextMultilineSendWhenTime = now; + beginningNextWhenTime = self.nextMultilineSendWhenTime; + #log( "** DEBUG: before: nextWhen: %d nextMultilineSendWhenTime: %d" % ( nextWhen, self.nextMultilineSendWhenTime )); for nodeOpennetRefKey in nodeOpennetRefKeys: self.after( nextWhen, self.privmsg, peernick, "opennetrefdirect %s=%s" % ( nodeOpennetRefKey, self.nodeOpennetRef[ nodeOpennetRefKey ] )) delay = random.randint(7,14) # 7-14 seconds between each line nextWhen += delay; - self.nextWhenTime += delay; + self.nextMultilineSendWhenTime += delay; self.after( nextWhen, self.privmsg, peernick, "opennetrefdirect End" ) - #log( "** DEBUG: after: nextWhen: %d nextWhenTime: %d beginningNextWhenTime: %d diff: %d" % ( nextWhen, self.nextWhenTime, beginningNextWhenTime, self.nextWhenTime - beginningNextWhenTime )); - #log( "** sendrefdirect(): sendRefDirectLock.release() after processing peernick: %s" % ( peernick )); - self.sendRefDirectLock.release(); # Lock shared between multiple threads for sending both darknet and opennet refs + #log( "** DEBUG: after: nextWhen: %d nextMultilineSendWhenTime: %d beginningNextWhenTime: %d diff: %d" % ( nextWhen, self.nextMultilineSendWhenTime, beginningNextWhenTime, self.nextMultilineSendWhenTime - beginningNextWhenTime )); + #log( "** sendrefdirect(): multilineSendLock.release() after processing peernick: %s" % ( peernick )); + self.multilineSendLock.release(); # Lock shared between all multiline senders #@-node:sendopennetrefdirect #@+node:process_any_identities_checked @@ -2251,26 +2248,26 @@ nodeDarknetRefKeys = self.nodeDarknetRef.keys() nodeDarknetRefKeys.sort() - #log( "** sendrefdirect(): sendRefDirectLock.acquire() before processing peernick: %s" % ( peernick )); - self.sendRefDirectLock.acquire( 1 ); # Lock shared between multiple threads for sending both darknet and opennet refs + #log( "** sendrefdirect(): multilineSendLock.acquire() before processing peernick: %s" % ( peernick )); + self.multilineSendLock.acquire( 1 ); # Lock shared between all multiline senders # Spread out the lines of the ref so we don't trigger the babbler detector of a receiving refbot nextWhen = 0; now = long( math.floor( time.time() )); - if( self.nextWhenTime > now): - nextWhen = self.nextWhenTime - now; + if( self.nextMultilineSendWhenTime > now): + nextWhen = self.nextMultilineSendWhenTime - now; else: - self.nextWhenTime = now; - beginningNextWhenTime = self.nextWhenTime; - #log( "** DEBUG: before: nextWhen: %d nextWhenTime: %d" % ( nextWhen, self.nextWhenTime )); + self.nextMultilineSendWhenTime = now; + beginningNextWhenTime = self.nextMultilineSendWhenTime; + #log( "** DEBUG: before: nextWhen: %d nextMultilineSendWhenTime: %d" % ( nextWhen, self.nextMultilineSendWhenTime )); for nodeDarknetRefKey in nodeDarknetRefKeys: self.after( nextWhen, self.privmsg, peernick, "refdirect %s=%s" % ( nodeDarknetRefKey, self.nodeDarknetRef[ nodeDarknetRefKey ] )) delay = random.randint(7,14) # 7-14 seconds between each line nextWhen += delay; - self.nextWhenTime += delay; + self.nextMultilineSendWhenTime += delay; self.after( nextWhen, self.privmsg, peernick, "refdirect End" ) - #log( "** DEBUG: after: nextWhen: %d nextWhenTime: %d beginningNextWhenTime: %d diff: %d" % ( nextWhen, self.nextWhenTime, beginningNextWhenTime, self.nextWhenTime - beginningNextWhenTime )); - #log( "** sendrefdirect(): sendRefDirectLock.release() after processing peernick: %s" % ( peernick )); - self.sendRefDirectLock.release(); # Lock shared between multiple threads for sending both darknet and opennet refs + #log( "** DEBUG: after: nextWhen: %d nextMultilineSendWhenTime: %d beginningNextWhenTime: %d diff: %d" % ( nextWhen, self.nextMultilineSendWhenTime, beginningNextWhenTime, self.nextMultilineSendWhenTime - beginningNextWhenTime )); + #log( "** sendrefdirect(): multilineSendLock.release() after processing peernick: %s" % ( peernick )); + self.multilineSendLock.release(); # Lock shared between all multiline senders #@-node:sendrefdirect #@+node:url_is_known_pastebin From zothar at freenetproject.org Sun Nov 11 15:21:19 2007 From: zothar at freenetproject.org (zothar at freenetproject.org) Date: Sun, 11 Nov 2007 15:21:19 +0000 (UTC) Subject: [Pyfreenet] r1