1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Basic locking functionality.
"""
import os
import errno
import fcntl
import tempfile
from chromite.lib import cros_build_lib
class _Lock(cros_build_lib.MasterPidContextManager):
"""Base lockf based locking. Derivatives need to override _GetFd"""
def __init__(self, description=None, verbose=True):
"""Initialize this instance.
Args:
path: On disk pathway to lock. Can be a directory or a file.
description: A description for this lock- what is it protecting?
"""
cros_build_lib.MasterPidContextManager.__init__(self)
self._verbose = verbose
self.description = description
self._fd = None
@property
def fd(self):
if self._fd is None:
self._fd = self._GetFd()
# Ensure that all derivatives of this lock can't bleed the fd
# across execs.
fcntl.fcntl(self._fd, fcntl.F_SETFD,
fcntl.fcntl(self._fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC)
return self._fd
def _GetFd(self):
raise NotImplementedError(self, '_GetFd')
def _enforce_lock(self, flags, message):
# Try nonblocking first, if it fails, display the context/message,
# and then wait on the lock.
try:
fcntl.lockf(self.fd, flags|fcntl.LOCK_NB)
return
except EnvironmentError as e:
if e.errno == errno.EDEADLOCK:
self.unlock()
elif e.errno != errno.EAGAIN:
raise
if self.description:
message = '%s: blocking while %s' % (self.description, message)
if self._verbose:
cros_build_lib.Info(message)
try:
fcntl.lockf(self.fd, flags)
except EnvironmentError as e:
if e.errno != errno.EDEADLOCK:
raise
self.unlock()
fcntl.lockf(self.fd, flags)
def read_lock(self, message="taking read lock"):
"""
Take a read lock (shared), downgrading from write if required.
Args:
message: A description of what/why this lock is being taken.
Returns:
self, allowing it to be used as a `with` target.
Raises:
IOError if the operation fails in some way.
"""
self._enforce_lock(fcntl.LOCK_SH, message)
return self
def write_lock(self, message="taking write lock"):
"""
Take a write lock (exclusive), upgrading from read if required.
Note that if the lock state is being upgraded from read to write,
a deadlock potential exists- as such we *will* release the lock
to work around it. Any consuming code should not assume that
transitioning from shared to exclusive means no one else has
gotten at the critical resource in between for this reason.
Args:
message: A description of what/why this lock is being taken.
Returns:
self, allowing it to be used as a `with` target.
Raises:
IOError if the operation fails in some way.
"""
self._enforce_lock(fcntl.LOCK_EX, message)
return self
def unlock(self):
"""
Release any locks held. Noop if no locks are held.
Raises:
IOError if the operation fails in some way.
"""
if self._fd is not None:
fcntl.lockf(self._fd, fcntl.LOCK_UN)
def __del__(self):
# TODO(ferringb): Convert this to snakeoil.weakref.WeakRefFinalizer
# if/when that rebasing occurs.
self.close()
def close(self):
"""
Release the underlying lock and close the fd.
"""
if self._fd is not None:
self.unlock()
os.close(self._fd)
self._fd = None
def _enter(self):
# Force the fd to be opened via touching the property.
# We do this to ensure that even if entering a context w/out a lock
# held, we can do locking in that critical section if the code requests it.
# pylint: disable=W0104
self.fd
return self
def _exit(self, exc_type, exc, traceback):
try:
self.unlock()
finally:
self.close()
class FileLock(_Lock):
def __init__(self, path, description=None, verbose=True):
"""
Args:
path: On disk pathway to lock. Can be a directory or a file.
description: A description for this lock- what is it protecting?
"""
if description is None:
description = "lock %s" % (path,)
_Lock.__init__(self, description=description, verbose=verbose)
self.path = os.path.abspath(path)
def _GetFd(self):
# If we're on py3.4 and this attribute is exposed, use it to close
# the threading race between open and fcntl setting; this is
# extremely paranoid code, but might as well.
cloexec = getattr(os, 'O_CLOEXEC', 0)
# There exist race conditions where the lock may be created by
# root, thus denying subsequent accesses from others. To prevent
# this, we create the lock with mode 0o666.
try:
value = os.umask(000)
fd = os.open(self.path, os.W_OK|os.O_CREAT|cloexec, 0o666)
finally:
os.umask(value)
return fd
class ProcessLock(_Lock):
"""Process level locking visible to parent/child only.
This lock is basically a more robust version of what
multiprocessing.Lock does. That implementation uses semaphores
internally which require cleanup/deallocation code to run to release
the lock; a SIGKILL hitting the process holding the lock violates those
assumptions leading to a stuck lock.
Thus this implementation is based around locking of a deleted tempfile;
lockf locks are guranteed to be released once the process/fd is closed.
"""
def _GetFd(self):
with tempfile.TemporaryFile() as f:
# We don't want to hold onto the object indefinitely; we just want
# the fd to a temporary inode, preferably one that isn't vfs accessible.
# Since TemporaryFile closes the fd once the object is GC'd, we just
# dupe the fd so we retain a copy, while the original TemporaryFile
# goes away.
return os.dup(f.fileno())
|
