"""High-level file operations."""
import errno
import logging
import os
import shutil
logger = logging.getLogger(__name__)
def is_safe_subdir_name(name) -> bool:
return (
isinstance(name, str)
and bool(name)
and "\x00" not in name
and name == os.path.basename(name)
and name not in (".", "..")
)
def _is_nfs_silly_rename(path):
"""NFS replaces an unlinked-but-still-open file with a sibling
`.nfsXXXX` entry that lingers until the holder closes it. The
name has the magic `.nfs` prefix; treating those as ignorable
EBUSY/ENOTEMPTY sources is what every long-running process on an
NFS-backed work dir ends up needing."""
return os.path.basename(str(path)).startswith(".nfs")
def _only_nfs_silly_renames_inside(path):
"""Return True if the directory at `path` exists and contains
nothing but `.nfsXXXX` survivors."""
try:
entries = os.listdir(path)
except OSError:
return False
if not entries:
return False
return all(_is_nfs_silly_rename(name) for name in entries)
def _swallow_nfs(func, path, exc_info):
"""`onerror` callback for shutil.rmtree.
NFS-backed work dirs (k8s shared volumes especially) routinely
contain `.nfsXXXX` silly-rename files left behind by a syncer
that still holds an open fd to a file we just unlinked. Two
shapes appear:
1. unlink(`<dir>/.nfsXXXX`) → EBUSY because the syncer still
has the fd open. The file expires on its own.
2. rmdir(`<dir>`) → ENOTEMPTY because the .nfsXXXX child is
still there. Same root cause.
Both are tolerable: leaving the dir + .nfs survivors in place is
not a leak — the holder eventually closes its fd and the next
cleanup pass succeeds.
Re-raise everything else.
"""
err = exc_info[1] if exc_info else None
if err is None:
return
if not isinstance(err, OSError) or err.errno not in (
errno.EBUSY,
errno.ENOTEMPTY,
):
raise err
if _is_nfs_silly_rename(path) or _only_nfs_silly_renames_inside(path):
logger.debug(
"ignoring NFS silly-rename leftover at %s during rmtree"
" (errno=%s)",
path,
err.errno,
)
return
raise err
def rmtree(path, ignore_errors=False, onerror=None, *, max_tries=3):
"""More robust shutil.rmtree.
Retry on "Directory not empty" race condition:
https://github.com/ansible/ansible/issues/34335#issuecomment-362995700
Also tolerate NFS silly-rename ``.nfsXXXX`` leftovers — they
expire when the holder closes its fd and there is nothing for
a synchronous cleanup to do that wouldn't race.
"""
# Compose onerror so callers' custom handlers still see other
# errors. We let our handler swallow only the NFS pathology.
# Only cluster deployments put work dirs on NFS; standalone keeps
# the caller's onerror untouched.
# Env read (not utils.is_cluster) to avoid a circular import.
in_cluster = os.environ.get("IS_IM_CLUSTER") == "1"
if not in_cluster:
effective_onerror = onerror
elif onerror is None:
effective_onerror = _swallow_nfs
else:
def _chained(func, path_, exc_info):
try:
_swallow_nfs(func, path_, exc_info)
except Exception:
onerror(func, path_, exc_info)
effective_onerror = _chained
retriable = [errno.EEXIST, errno.ENOTEMPTY]
if in_cluster:
# An EBUSY that got past _swallow_nfs is a busy path on a shared
# volume, where the holder usually lets go within a retry.
retriable.append(errno.EBUSY)
for i in range(1, max_tries + 1):
try:
return shutil.rmtree(path, ignore_errors, effective_onerror)
except OSError as e:
if i == max_tries or e.errno not in retriable:
raise
logger.warning("Can't remove %s tree, reason: %s", path, e)
|