python.larbatch_posix Namespace Reference

Classes
class	dcache_file

Functions
def	use_grid (force=True)
def	open (path, mode='r', buf=-1)
def	readlines (path)
def	copy (src, dest)
def	listdir (path)
def	exists (path)
def	isdir (path)
def	stat (path)
def	access (path, mode)
def	walk (top, topdown=True)
def	mkdir (path, mode=0o777)
def	makedirs (path, mode=0o777)
def	rename (src, dest)
def	remove (path)
def	rmdir (path)
def	rmtree (path)
def	chmod (path, mode)
def	symlink (src, dest)
def	readlink (path)
def	root_stream (path)

Variables
	pnfs_is_mounted = os.path.isdir('/pnfs')
string	prefer_grid = 'LARBATCH_GRID'
string	debug = 'LARBATCH_DEBUG'

Function Documentation

def python.larbatch_posix.access	(		path,
			mode
	)

Definition at line 538 of file larbatch_posix.py.

def access(path, mode):

    result = False
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Check access for %s using ifdh.' % path)
        sr = stat(path)
        if sr.st_mode != 0:

            # File exists.

            result = True

            # Test permissions.

            if mode & os.R_OK:
                if not sr.st_mode & statmod.S_IRUSR:
                    result = False
            if mode & os.W_OK:
                if not sr.st_mode & statmod.S_IWUSR:
                    result = False
            if mode & os.X_OK:
                if not sr.st_mode & statmod.S_IXUSR:
                    result = False

    else:
        if debug:
            print('*** Larbatch_posix: Check access for %s using posix.' % path)
        result = os.access(path, mode)

    # Done.

    return result


# Walk directory tree.  Like os.walk, this function returns an iterator over
# 3-tuples, one for each directory in the tree rooted in the specified
# top directory.  Each 3-tuple contains the following information.
#
# 1.  Path of directory.
# 2.  List of dictory names in this directory.
# 3.  List of non-directory files in this directory.
#
# In case of posix mode, this function includes its own implementation
# of the walking algorithm so that we can take advantage of optimzations
# contained in this module's implementation of isdir.

def python.larbatch_posix.chmod	(		path,
			mode
	)

Definition at line 804 of file larbatch_posix.py.

def chmod(path, mode):
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Change mode for %s using ifdh.' % path)
        larbatch_utilities.ifdh_chmod(path, mode)
    else:
        if debug:
            print('*** Larbatch_posix: Change mode for %s using posix.' % path)
        os.chmod(path, mode)


# Make symbolic link.
# Use nfs.

def python.larbatch_posix.copy	(		src,
			dest
	)

Definition at line 294 of file larbatch_posix.py.

def copy(src, dest):
    if exists(dest):
        remove(dest)
    if (src.startswith('/pnfs/') or dest.startswith('/pnfs/')):
        if prefer_grid or not pnfs_is_mounted:
            if debug:
                print('*** Larbatch_posix: Copy %s to %s using ifdh.' % (src, dest))
            larbatch_utilities.ifdh_cp(src, dest)
        else:
            if debug:
                print('*** Larbatch_posix: Copy %s to %s using posix with timeout.' % (src, dest))
            larbatch_utilities.posix_cp(src, dest)
    else:
        if debug:
            print('*** Larbatch_posix: Copy %s to %s using posix.' % (src, dest))
        shutil.copy(src, dest)

    # Done

    return


# List directory contents.

def python.larbatch_posix.exists

(

path

)

Definition at line 379 of file larbatch_posix.py.

def exists(path):

    result = False
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Check existence of %s using ifdh.' % path)

        # Do "ifdh ls."

        try:
            larbatch_utilities.ifdh_ls(path, 0)
            result = True
        except:
            result = False

    else:
        if debug:
            print('*** Larbatch_posix: Check existence of %s using posix.' % path)
        #result = os.path.exists(path)

        # In order to reduce hang risk from stat'ing file,
        # check existence by getting contents of parent directory.

        npath = os.path.normpath(path)     # Strip trailing '/'.
        dir = os.path.dirname(npath)
        base = os.path.basename(npath)
        if dir == '':
            dir = '.'
        if isdir(dir):
            files = listdir(dir)
            for filename in files:
                if base == filename:
                    result = True

    # Done.

    return result


# Test existence if directory.

def python.larbatch_posix.isdir

(

path

)

Definition at line 420 of file larbatch_posix.py.

def isdir(path):

    result = False

    # Optimizations for speed and to reduce hang risk by not stat'ing every file.

    if path[-5:] == '.list' or \
            path[-5:] == '.root' or \
            path[-5:] == '.json' or \
            path[-4:] == '.txt' or \
            path[-4:] == '.fcl' or \
            path[-4:] == '.out' or \
            path[-4:] == '.err' or \
            path[-3:] == '.sh' or \
            path[-5:] == '.stat':
        return False

    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Check existence of directory %s using ifdh.' % path)

        # Make sure path exists before trying to determine if it is a directory.

        if exists(path):

            # The only reliable way to get information about a directory is 
            # to get a listing of the parent directory.

            npath = os.path.normpath(path)      # Strip trailing '/'
            name = os.path.basename(npath)
            dir = os.path.dirname(npath)
            lines = larbatch_utilities.ifdh_ll(dir, 1)
            for line in lines:
                words = line.split()
                if len(words) > 5 and words[-1] == name:
                    if words[0][0] == 'd':
                        result = True


    else:
        if debug:
            print('*** Larbatch_posix: Check existence of directory %s using posix.' % path)
        result = os.path.isdir(path)

    # Done.

    return result


# Get file status.  
#
# This function is a partial emulation of os.stat.  The return value
# is a type os.stat_result, which is a 10-tuple of values.  The following
# values are filled by this function, since information is not always
# available from grid tools.
#
# mode  - File type and permissions.
# uid   - Owner uid.  For compatibility, always set to the process uid.
# gid   - Owner gid.  For compatibility, always set to the process gid.
# nlink - Number of links.
# size  - Object size.

def python.larbatch_posix.listdir

(

path

)

Definition at line 318 of file larbatch_posix.py.

def listdir(path):

    if not isdir(path):
        raise OSError('%s is not a directory.' % path)
    result = []
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Listdir %s using ifdh.' % path)

        # Get normalized tail.

        tail = os.path.normpath(path[-6:])

        # Call "ifdh ls".

        contents = larbatch_utilities.ifdh_ls(path, 1)

        # Loop over contents returned by ifdh.
        # Normalize the paths returned by "ifdh ls", which in this context mainly
        # has the effect of stripping off trailing '/' on directories.
        # Filter out parent directory, which ifdh sometimes (usually?) includes in result.

        for c in contents:
            nc = os.path.normpath(c.strip())
            if not nc.endswith(tail):
                result.append(os.path.basename(nc))

    else:
        if debug:
            print('*** Larbatch_posix: Listdir %s using posix.' % path)
        #result = os.listdir(path)

        # To reduce hang risk, read contents of directory in a subprocess with
        # a timeout.

        cmd = ['ls', path]
        jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        q = queue.Queue()
        thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
        thread.start()
        thread.join(timeout=60)
        if thread.is_alive():
            if debug:
                print('*** Larbatch_posix: Terminating subprocess.')
            jobinfo.terminate()
            thread.join()
        rc = q.get()
        jobout = convert_str(q.get())
        joberr = convert_str(q.get())
        if rc == 0:
            for word in jobout.split():
                result.append(word)

    # Done.

    return result


# Test existence.  Works for files and directories.

def python.larbatch_posix.makedirs	(		path,
			mode = 0o777
	)

Definition at line 658 of file larbatch_posix.py.

def makedirs(path, mode=0o777):
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Make directory recursively for %s using ifdh.' % path)

        # Make sure parent directory exists.

        np = os.path.normpath(path)    # Stip trailing '/', if present.
        parent = os.path.dirname(np)
        if not isdir(parent):
            makedirs(parent, mode)

        # Now make directory itself.

        larbatch_utilities.ifdh_mkdir(path)
    else:
        if debug:
            print('*** Larbatch_posix: Make directory recursively for %s using posix.' % path)
        os.makedirs(path, mode)


# Rename file.
# "ifdh mv" seems to be buggy, so use uberftp.

def python.larbatch_posix.mkdir	(		path,
			mode = 0o777
	)

Definition at line 642 of file larbatch_posix.py.

def mkdir(path, mode=0o777):
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Make directory for %s using ifdh.' % path)
        larbatch_utilities.ifdh_mkdir(path)
    else:
        if debug:
            print('*** Larbatch_posix: Make directory for %s using posix.' % path)
        os.mkdir(path, mode)


# Make directory and parents.
# Mode argument is ignored for dCache files, but is passed to os.mkdir
# for non-dCache files.
# "ifdh mkdir_p" is buggy, so we do the recursion locally.

def python.larbatch_posix.open	(		path,
			mode = 'r',
			buf = -1
	)

Definition at line 275 of file larbatch_posix.py.

def open(path, mode='r', buf=-1):
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Opening dcache_file %s using mode %s.' % (path, mode))
        return dcache_file(path, mode, buf)
    else:
        if debug:
            print('*** Larbatch_posix: Opening normal file %s using mode %s.' % (path, mode))
        return __builtins__['open'](path, mode, buf)


# Read lines from a file.

def python.larbatch_posix.readlines

(

path

)

Definition at line 288 of file larbatch_posix.py.

def readlines(path):
    return open(path).readlines()


# Copy file.

def python.larbatch_posix.readlink

(

path

)

Definition at line 853 of file larbatch_posix.py.

def readlink(path):

    result = ''

    # Make sure we have a kerberos ticket.

    if path.startswith('/pnfs/') and not_pnfs_is_mounted:
        if debug:
            print('*** Larbatch_posix: Read symbolic link %s using nfs server.' % path)
        larbatch_utilities.test_ticket()
        cmd = ['ssh', larbatch_utilities.nfs_server(), 'readlink', path]
        jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        q = queue.Queue()
        thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
        thread.start()
        thread.join(timeout=60)
        if thread.is_alive():
            if debug:
                print('*** Larbatch_posix: Terminating subprocess.')
            jobinfo.terminate()
            thread.join()
        rc = q.get()
        jobout = convert_str(q.get())
        joberr = convert_str(q.get())
        if rc != 0:
            raise IFDHError(cmd, rc, jobout, joberr)
        result = jobout.strip()

    else:
        if debug:
            print('*** Larbatch_posix: Read symbolic link %s using posix.' % path)
        result = os.readlink(path)

    # Done.

    return result


# Convert a root file path to a streamable path or uri that can be opened 
# using TFile::Open.
# Non-dCache paths (paths not starting with '/pnfs/') are not changed.
# dCache paths (patsh starting with '/pnfs/') may be converted to an xrootd uri.

def python.larbatch_posix.remove

(

path

)

Definition at line 715 of file larbatch_posix.py.

def remove(path):
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Delete file %s using ifdh.' % path)
        larbatch_utilities.ifdh_rm(path)
    else:
        if debug:
            print('*** Larbatch_posix: Delete file %s using posix.' % path)

        # Deleting a file is a hang risk, especially, but not only, in dCache.
        # Therefore, use the following procedure.
        # 
        # 1.  Rename file to a random name (this can usually be done, even
        #     for undeletable files).
        #
        # 2.  Delete renamed file in a subprocess.  No need to wait for
        #     subprocess to finish, or check its exit status.

        #os.remove(path)
        newpath = path + '_' + str(uuid.uuid4())
        try:
            os.rename(path, newpath)
            os.system('rm -f %s &' % newpath)
            return
        except:
            pass
        os.remove(path)

# Delete empty directory.

def python.larbatch_posix.rename	(		src,
			dest
	)

Definition at line 682 of file larbatch_posix.py.

def rename(src, dest):
    if (src.startswith('/pnfs/') or
        dest.startswith('/pnfs/')) and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Rename %s to %s using ifdh.' % (src, dest))

        src_uri = larbatch_utilities.gridftp_uri(src)
        dest_path = larbatch_utilities.dcache_path(dest)
        cmd = ['uberftp', '-rename', src_uri, dest_path]
        jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        q = queue.Queue()
        thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
        thread.start()
        thread.join(timeout=60)
        if thread.is_alive():
            if debug:
                print('*** Larbatch_posix: Terminating subprocess.')
            jobinfo.terminate()
            thread.join()
        rc = q.get()
        jobout = convert_str(q.get())
        joberr = convert_str(q.get())
        if rc != 0:
            raise IFDHError(cmd, rc, jobout, joberr)
    else:
        if debug:
            print('*** Larbatch_posix: Rename %s to %s using posix.' % (src, dest))
        os.rename(src, dest)


# Delete file.

def python.larbatch_posix.rmdir

(

path

)

Definition at line 745 of file larbatch_posix.py.

def rmdir(path):
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Delete directoroy %s using ifdh.' % path)
        larbatch_utilities.ifdh_rmdir(path)
    else:
        if debug:
            print('*** Larbatch_posix: Delete directoroy %s using posix.' % path)
        os.rmdir(path)


# Delete directory tree.

def python.larbatch_posix.rmtree

(

path

)

Definition at line 758 of file larbatch_posix.py.

def rmtree(path):
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Delete directoroy tree %s using ifdh.' % path)

        # Delete contents recursively.

        lines = larbatch_utilities.ifdh_ll(path, 1)
        for line in lines:
            words = line.split()
            if len(words) > 5:
                if words[0][0] == 'd':
                    rmtree(os.path.join(path, words[-1]))
                else:
                    remove(os.path.join(path, words[-1]))

        # Directory should be empty when we get to here.

        rmdir(path)

    else:
        if debug:
            print('*** Larbatch_posix: Delete directoroy tree %s using posix.' % path)

        # Deleting a directory tree is a hang risk, especially, but not only, in dCache.
        # Therefore, use the following procedure.
        # 
        # 1.  Rename directory to a random name (this can usually be done, even
        #     for undeletable directories).
        #
        # 2.  Delete renamed directory in a subprocess.  No need to wait for
        #     subprocess to finish, or check its exit status.

        #shutil.rmtree(path)
        npath = os.path.normpath(path)              # Strip trailing '/'
        newpath = npath + '_' + str(uuid.uuid4())
        os.rename(npath, newpath)
        os.system('rm -rf %s &' % newpath)

    # Done

    return


# Change mode.

def python.larbatch_posix.root_stream

(

path

)

Definition at line 897 of file larbatch_posix.py.

def root_stream(path):

    stream = path
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Stream path %s using xrootd.' % path)
        larbatch_utilities.test_proxy()
        stream = larbatch_utilities.xrootd_uri(path)
    else:
        if debug:
            print('*** Larbatch_posix: Stream path %s as normal file.' % path)
    return stream

def python.larbatch_posix.stat

(

path

)

Definition at line 482 of file larbatch_posix.py.

def stat(path):

    result = None
    if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Stat %s using ifdh.' % path)

        # The only reliable way to get information about a directory is 
        # to get a listing of the parent directory.

        npath = os.path.normpath(path)      # Strip trailing '/'
        name = os.path.basename(npath)
        dir = os.path.dirname(npath)
        lines = larbatch_utilities.ifdh_ll(dir, 1)
        for line in lines:
            words = line.split()
            if len(words) >5 and words[-1] == name:

                # Found thie path.  Fill result.

                # Interpret mode string.

                mode = larbatch_utilities.parse_mode(words[0])

                # Get remaining fields.
                    
                nlinks = int(words[1])
                size = int(words[4])
                result = os.stat_result((mode,         # Mode
                                         0,            # Inode
                                         0,            # Device
                                         nlinks,       # Number of links
                                         os.getuid(),  # Uid
                                         os.getgid(),  # Gid
                                         size,         # Size
                                         0,            # Access time
                                         0,            # Mod time
                                         0))           # Creation time

    else:
        if debug:
            print('*** Larbatch_posix: Stat %s using posix.' % path)
        result = os.stat(path)

    if result == None:
        raise OSError('No such file or directory.')

    # Done.

    return result


# Test file access.
# This implementation only tests access to dCache files via the
# user permission, which is a limitation of grid tools.

def python.larbatch_posix.symlink	(		src,
			dest
	)

Definition at line 818 of file larbatch_posix.py.

def symlink(src, dest):

    # Make sure we have a kerberos ticket.

    if src.startswith('/pnfs/') and not pnfs_is_mounted:
        if debug:
            print('*** Larbatch_posix: Make symbolic link from %s to %s using nfs server.' % (src, dest))
        larbatch_utilities.test_ticket()
        cmd = ['ssh', larbatch_utilities.nfs_server(), 'ln', '-s', src, dest]
        jobinfo = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        q = queue.Queue()
        thread = threading.Thread(target=larbatch_utilities.wait_for_subprocess, args=[jobinfo, q])
        thread.start()
        thread.join(timeout=60)
        if thread.is_alive():
            if debug:
                print('*** Larbatch_posix: Terminating subprocess.')
            jobinfo.terminate()
            thread.join()
        rc = q.get()
        jobout = convert_str(q.get())
        joberr = convert_str(q.get())
        if rc != 0:
            raise IFDHError(cmd, rc, jobout, joberr)

    else:
        if debug:
            print('*** Larbatch_posix: Make symbolic link from %s to %s using posix.' % (src, dest))
        os.symlink(src, dest)


# Read a symbolic link.
# Use nfs.

def python.larbatch_posix.use_grid

(

force = True

)

Definition at line 150 of file larbatch_posix.py.

def use_grid(force=True):
    prefer_grid = force


# File-like class for dCache files.

def python.larbatch_posix.walk	(		top,
			topdown = True
	)

Definition at line 585 of file larbatch_posix.py.

def walk(top, topdown=True):

    # Quit if top directory doesn't exist.

    if not exists(top):
        return

    # Get contents of top directory using either ifdh or posix.

    dirs = []
    files = []
    if top.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
        if debug:
            print('*** Larbatch_posix: Walk directory tree for %s using ifdh.' % top)

        # Retrieve the contents of this directory using ifdh.

        lines = larbatch_utilities.ifdh_ll(top, 1)
        for line in lines:
            words = line.split()
            if len(words) > 5 and words[-1] != '__pycache__':
                if words[0][0] == 'd':
                    dirs.append(words[-1])
                else:
                    files.append(words[-1])
    else:
        if debug:
            print('*** Larbatch_posix: Walk directory tree for %s using posix.' % top)
        contents = listdir(top)
        for obj in contents:
            if obj != '__pycache__':
                if isdir(os.path.join(top, obj)):
                    dirs.append(obj)
                else:
                    files.append(obj)

    if topdown:
        yield top, dirs, files

    # Recursively descend into subdirectories.

    for dir in dirs:
        for result in walk(os.path.join(top, dir), topdown):
            yield result

    if not topdown:
        yield top, dirs, files

    # Done.

    return


# Make directory (parent directory must exist).
# Mode argument is ignored for dCache files, but is passed to os.mkdir
# for non-dCache files.

Variable Documentation

string python.larbatch_posix.debug = 'LARBATCH_DEBUG'

Definition at line 142 of file larbatch_posix.py.

python.larbatch_posix.pnfs_is_mounted = os.path.isdir('/pnfs')

Definition at line 140 of file larbatch_posix.py.

string python.larbatch_posix.prefer_grid = 'LARBATCH_GRID'

Definition at line 141 of file larbatch_posix.py.

---

Generated by   1.8.11