aboutsummaryrefslogtreecommitdiff
path: root/google
diff options
context:
space:
mode:
authorPeter Lamut <plamut@users.noreply.github.com>2020-06-09 20:48:10 +0200
committerGitHub <noreply@github.com>2020-06-09 18:48:10 +0000
commit74e0b0f8387207933c120af15b2bb5d175dd8f84 (patch)
treee8fe0a71fbfa6d2f59dece58a6b210f620111084 /google
parent945bafc47cfaf79f0947c970dba17dc8fbe3e197 (diff)
downloadpython-api-core-74e0b0f8387207933c120af15b2bb5d175dd8f84.tar.gz
feat: allow disabling response stream pre-fetch (#30)
Closes #25. This PR adds the ability to disable automatically pre-fetching the first item of a stream returned by `*-Stream` gRPC callables. This hook will be used in PubSub to fix the [stalled stream issue](https://github.com/googleapis/python-pubsub/issues/93), while also not affecting Firestore, since the default behavior is preserved. I realize the fix is far from ideal, but it's the least ugly among the approaches I tried, e.g. somehow passing the flag through `ResumableBidiRpc` (it's a messy rabbit hole). On the PubSub side monkeypatching the generated SubscriberClient will be needed, but it's a (relatively) clean one-liner: ```patch diff --git google/cloud/pubsub_v1/gapic/subscriber_client.py google/cloud/pubsub_v1/gapic/subscriber_client.py index e98a686..1d6c058 100644 --- google/cloud/pubsub_v1/gapic/subscriber_client.py +++ google/cloud/pubsub_v1/gapic/subscriber_client.py @@ -1169,6 +1169,8 @@ class SubscriberClient(object): default_timeout=self._method_configs["StreamingPull"].timeout, client_info=self._client_info, ) + # TODO: explain this monkeypatch! + self.transport.streaming_pull._prefetch_first_result_ = False return self._inner_api_calls["streaming_pull"]( requests, retry=retry, timeout=timeout, metadata=metadata ``` If/when we merge this, we should also release it, and then we can add `!= 1.17.0` to the `google-api-core` version pin in PubSub. ### PR checklist - [x] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-api-core/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [x] Ensure the tests and linter pass - [x] Code coverage does not decrease (if any source code was changed) - [x] Appropriate docs were updated (if necessary)
Diffstat (limited to 'google')
-rw-r--r--google/api_core/grpc_helpers.py12
1 files changed, 9 insertions, 3 deletions
diff --git a/google/api_core/grpc_helpers.py b/google/api_core/grpc_helpers.py
index fde6c33..b617ddf 100644
--- a/google/api_core/grpc_helpers.py
+++ b/google/api_core/grpc_helpers.py
@@ -62,14 +62,15 @@ def _wrap_unary_errors(callable_):
class _StreamingResponseIterator(grpc.Call):
- def __init__(self, wrapped):
+ def __init__(self, wrapped, prefetch_first_result=True):
self._wrapped = wrapped
# This iterator is used in a retry context, and returned outside after init.
# gRPC will not throw an exception until the stream is consumed, so we need
# to retrieve the first result, in order to fail, in order to trigger a retry.
try:
- self._stored_first_result = six.next(self._wrapped)
+ if prefetch_first_result:
+ self._stored_first_result = six.next(self._wrapped)
except TypeError:
# It is possible the wrapped method isn't an iterable (a grpc.Call
# for instance). If this happens don't store the first result.
@@ -141,7 +142,12 @@ def _wrap_stream_errors(callable_):
def error_remapped_callable(*args, **kwargs):
try:
result = callable_(*args, **kwargs)
- return _StreamingResponseIterator(result)
+ # Auto-fetching the first result causes PubSub client's streaming pull
+ # to hang when re-opening the stream, thus we need examine the hacky
+ # hidden flag to see if pre-fetching is disabled.
+ # https://github.com/googleapis/python-pubsub/issues/93#issuecomment-630762257
+ prefetch_first = getattr(callable_, "_prefetch_first_result_", True)
+ return _StreamingResponseIterator(result, prefetch_first_result=prefetch_first)
except grpc.RpcError as exc:
six.raise_from(exceptions.from_grpc_error(exc), exc)