From pmpp at freenetproject.org Mon Apr 14 18:20:52 2008 From: pmpp at freenetproject.org (pmpp at freenetproject.org) Date: Mon, 14 Apr 2008 18:20:52 +0000 (UTC) Subject: [Pyfreenet] r19332 - trunk/apps/pyFreenet/fcp Message-ID: <20080414182052.7D1C2479E18@freenetproject.org> Author: pmpp Date: 2008-04-14 18:20:52 +0000 (Mon, 14 Apr 2008) New Revision: 19332 Modified: trunk/apps/pyFreenet/fcp/node.py trunk/apps/pyFreenet/fcp/put.py Log: added command line switches to fcpput and code to use uncompressed inserts and direct disk access Modified: trunk/apps/pyFreenet/fcp/node.py =================================================================== --- trunk/apps/pyFreenet/fcp/node.py 2008-04-14 17:46:08 UTC (rev 19331) +++ trunk/apps/pyFreenet/fcp/node.py 2008-04-14 18:20:52 UTC (rev 19332) @@ -633,8 +633,8 @@ opts['MaxRetries'] = kw.get("maxretries", -1) opts['PriorityClass'] = kw.get("priority", 3) opts['GetCHKOnly'] = chkOnly - opts['DontCompress'] = toBool(kw.get("nocompress", "false")) - + opts['DontCompress'] = toBool(kw.get("nocompress", "false")) + if kw.has_key("file"): opts['UploadFrom'] = "disk" opts['Filename'] = kw['file'] @@ -1263,7 +1263,8 @@ readFileContents = readFile.read(); readFile.close(); kw[ 'ReadFilename' ] = readFilename; - kw[ 'ReadContent' ] = readFileContents; + kw[ 'ReadContent' ] = readFileContents; + if( requestResult.has_key( 'WriteFilename' ) and requestResult.has_key( 'ContentToWrite' )): writeFilename = requestResult[ 'WriteFilename' ]; contentToWrite = requestResult[ 'ContentToWrite' ]; @@ -1272,10 +1273,11 @@ writeFile.close(); writeFileStatObject = os.stat( writeFilename ); writeFileMode = writeFileStatObject.st_mode; - os.chmod( writeFilename, writeFileMode | stat.S_IREAD | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH ); + os.chmod( writeFilename, writeFileMode | stat.S_IREAD | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH ); + responseResult = self._submitCmd("__global", "TestDDAResponse", **kw) if( None != writeFilename ): - try: + try: os.remove( writeFilename ); except OSError, msg: pass; @@ -2188,8 +2190,8 @@ return if hdr == 'PutFetchable': - uri = msg['URI'] - job.kw['URI'] = uri + uri = msg['URI'] + job.kw['URI'] = uri job.callback('pending', msg) return @@ -2297,11 +2299,11 @@ job._putResult(job.msgs) return - if hdr == 'PersistentRequestRemoved': - if self.jobs.has_key(id): - del self.jobs[id] - return - + if hdr == 'PersistentRequestRemoved': + if self.jobs.has_key(id): + del self.jobs[id] + return + # ----------------------------- # handle testDDA messages Modified: trunk/apps/pyFreenet/fcp/put.py =================================================================== --- trunk/apps/pyFreenet/fcp/put.py 2008-04-14 17:46:08 UTC (rev 19331) +++ trunk/apps/pyFreenet/fcp/put.py 2008-04-14 18:20:52 UTC (rev 19332) @@ -59,7 +59,12 @@ print " The mimetype under which to insert the key. If not given, then" print " an attempt will be made to guess it from the filename. If no" print " filename is given, or if this attempt fails, the mimetype" - print " 'text/plain' will be used as a fallback" + print " 'text/plain' will be used as a fallback" + print " -c, --compress" + print " Enable compression of inserted data (default is no compression)" + print " -d, --disk" + print " Try to have the node access file on disk directly , it will try then a fallback is provided" + print " nb:give the path relative to node filesystem not from where you're running this program" print " -p, --persistence=" print " Set the persistence type, one of 'connection', 'reboot' or 'forever'" print " -g, --global" @@ -77,7 +82,6 @@ print " Instead of specifying -H and/or -P, you can define the environment" print " variables FCP_HOST and/or FCP_PORT respectively" - sys.exit(0) #@-node:help #@+node:main @@ -98,15 +102,15 @@ "persistence" : "connection", "async" : False, "priority" : 3, - "MaxRetries" : -1, + "MaxRetries" : -1, } # process command line switches try: cmdopts, args = getopt.getopt( sys.argv[1:], - "?hvH:P:m:gp:nr:t:V", - ["help", "verbose", "fcpHost=", "fcpPort=", "mimetype=", "global", + "?hvH:P:m:gcdp:nr:t:V", + ["help", "verbose", "fcpHost=", "fcpPort=", "mimetype=", "global","compress","disk", "persistence=", "nowait", "priority=", "timeout=", "version", ] @@ -117,17 +121,21 @@ sys.exit(2) output = None verbose = False - #print cmdopts + #print cmdopts + + makeDDARequest=False + opts['nocompress'] = True + for o, a in cmdopts: - - if o in ("-?", "-h", "--help"): - help() - if o in ("-V", "--version"): print "This is %s, version %s" % (progname, node.fcpVersion) - sys.exit(0) - - if o in ("-v", "--verbosity"): + sys.exit(0) + + elif o in ("-?", "-h", "--help"): + help() + sys.exit(0) + + elif o in ("-v", "--verbosity"): if verbosity < node.DETAIL: verbosity = node.DETAIL else: @@ -135,31 +143,38 @@ opts['Verbosity'] = 1023 verbose = True - if o in ("-H", "--fcpHost"): + elif o in ("-H", "--fcpHost"): fcpHost = a - if o in ("-P", "--fcpPort"): + elif o in ("-P", "--fcpPort"): try: fcpPort = int(a) except: usage("Invalid fcpPort argument %s" % repr(a)) - if o in ("-m", "--mimetype"): - mimetype = a - - if o in ("-p", "--persistence"): + elif o in ("-m", "--mimetype"): + mimetype = a + + elif o in ("-c", "--compress"): + opts['nocompress'] = False + + elif o in ("-d","--disk"): + makeDDARequest=True + + + elif o in ("-p", "--persistence"): if a not in ("connection", "reboot", "forever"): usage("Persistence must be one of 'connection', 'reboot', 'forever'") opts['persistence'] = a - if o in ("-g", "--global"): + elif o in ("-g", "--global"): opts['Global'] = "true" - if o in ("-n", "--nowait"): + elif o in ("-n", "--nowait"): opts['async'] = True nowait = True - if o in ("-r", "--priority"): + elif o in ("-r", "--priority"): try: pri = int(a) if pri < 0 or pri > 6: @@ -168,7 +183,7 @@ usage("Invalid priority '%s'" % pri) opts['priority'] = int(a) - if o in ("-t", "--timeout"): + elif o in ("-t", "--timeout"): try: timeout = node.parseTime(a) except: @@ -212,35 +227,60 @@ traceback.print_exc(file=sys.stderr) usage("Failed to connect to FCP service at %s:%s" % (fcpHost, fcpPort)) - # grab the data - if not infile: - data = sys.stdin.read() - else: + + TestDDARequest=False + + if makeDDARequest: + if infile is not None: + ddareq=dict() + ddafile = os.path.abspath(infile) + + ddareq["Directory"]= os.path.dirname(ddafile) + ddareq["WantReadDirectory"]="True" + ddareq["WantWriteDirectory"]="false" + print "Absolute filepath used for node direct disk access :",ddareq["Directory"] + print "File to insert :",os.path.basename( ddafile ) + TestDDARequest=n.testDDA(**ddareq) + print "Result of dda request :",TestDDARequest + + if TestDDARequest: + opts["file"]=ddafile + uri = n.put(uri,**opts) + else: + sys.stderr.write("%s: disk access failed to insert file %s fallback to direct\n" % (progname,ddafile) ) + else: + sys.stderr.write("%s: disk access need a disk filename\n" % progname ) + + # try to insert the key using "direct" way if dda has failed + if not TestDDARequest: + # grab the data + if not infile: + data = sys.stdin.read() + else: + try: + data = file(infile, "rb").read() + except: + n.shutdown() + usage("Failed to read input from file %s" % repr(infile)) + try: - data = file(infile, "rb").read() + #print "opts=%s" % str(opts) + uri = n.put(uri, data=data, **opts) except: + if verbose: + traceback.print_exc(file=sys.stderr) n.shutdown() - usage("Failed to read input from file %s" % repr(infile)) + sys.stderr.write("%s: Failed to insert key %s\n" % (progname, repr(uri))) + sys.exit(1) + + if nowait: + # got back a job ticket, wait till it has been sent + uri.waitTillReqSent() + else: + # successful, return the uri + sys.stdout.write(uri) + sys.stdout.flush() - # try to insert the key - try: - #print "opts=%s" % str(opts) - uri = n.put(uri, data=data, **opts) - except: - if verbose: - traceback.print_exc(file=sys.stderr) - n.shutdown() - sys.stderr.write("%s: Failed to insert key %s\n" % (progname, repr(uri))) - sys.exit(1) - - if nowait: - # got back a job ticket, wait till it has been sent - uri.waitTillReqSent() - else: - # successful, return the uri - sys.stdout.write(uri) - sys.stdout.flush() - n.shutdown() # all done