feat: add support for Etag headers on reads (#489)

Support conditional requests based on ETag for read operations (`reload`, `exists`, `download_*`). My own testing seems to indicate that the JSON API does not support ETag If-Match/If-None-Match headers on modify requests (`patch`, `delete`, etc.), please correct me if I am mistaken.

This part two of #451. Part one in #488.
Fixes #451 🦕

Author: daniellehanks

docs/generation_metageneration.rst

@@ -1,17 +1,31 @@
-Conditional Requests Via Generation / Metageneration Preconditions
-==================================================================
+Conditional Requests Via ETag / Generation / Metageneration Preconditions
+=========================================================================
 
 Preconditions tell Cloud Storage to only perform a request if the
-:ref:`generation <concept-generation>` or
+:ref:`ETag <concept-etag>`, :ref:`generation <concept-generation>`, or
 :ref:`metageneration <concept-metageneration>` number of the affected object
-meets your precondition criteria. These checks of the generation and
+meets your precondition criteria. These checks of the ETag, generation, and
 metageneration numbers ensure that the object is in the expected state,
 allowing you to perform safe read-modify-write updates and conditional
 operations on objects
 
 Concepts
 --------
 
+.. _concept-etag:
+
+ETag
+::::::::::::::
+
+An ETag is returned as part of the response header whenever a resource is
+returned, as well as included in the resource itself. Users should make no
+assumptions about the value used in an ETag except that it changes whenever the
+underlying data changes, per the
+`specification <https://tools.ietf.org/html/rfc7232#section-2.3>`_
+
+The ``ETag`` attribute is set by the GCS back-end, and is read-only in the
+client library.
+
 .. _concept-metageneration:
 
 Metageneration
@@ -59,6 +73,32 @@ See also
 Conditional Parameters
 ----------------------
 
+.. _using-if-etag-match:
+
+Using ``if_etag_match``
+:::::::::::::::::::::::::::::
+
+Passing the ``if_etag_match`` parameter to a method which retrieves a
+blob resource (e.g.,
+:meth:`Blob.reload <google.cloud.storage.blob.Blob.reload>`)
+makes the operation conditional on whether the blob's current ``ETag`` matches
+the given value. This parameter is not supported for modification (e.g.,
+:meth:`Blob.update <google.cloud.storage.blob.Blob.update>`).
+
+
+.. _using-if-etag-not-match:
+
+Using ``if_etag_not_match``
+:::::::::::::::::::::::::::::
+
+Passing the ``if_etag_not_match`` parameter to a method which retrieves a
+blob resource (e.g.,
+:meth:`Blob.reload <google.cloud.storage.blob.Blob.reload>`)
+makes the operation conditional on whether the blob's current ``ETag`` matches
+the given value. This parameter is not supported for modification (e.g.,
+:meth:`Blob.update <google.cloud.storage.blob.Blob.update>`).
+
+
 .. _using-if-generation-match:
 
 Using ``if_generation_match``

google/cloud/storage/_helpers.py

@@ -22,6 +22,7 @@
 from datetime import datetime
 import os
 
+from six import string_types
 from six.moves.urllib.parse import urlsplit
 from google import resumable_media
 from google.cloud.storage.constants import _DEFAULT_TIMEOUT
@@ -34,6 +35,12 @@
 
 _DEFAULT_STORAGE_HOST = u"https://storage.googleapis.com"
 
+# etag match parameters in snake case and equivalent header
+_ETAG_MATCH_PARAMETERS = (
+    ("if_etag_match", "If-Match"),
+    ("if_etag_not_match", "If-None-Match"),
+)
+
 # generation match parameters in camel and snake cases
 _GENERATION_MATCH_PARAMETERS = (
     ("if_generation_match", "ifGenerationMatch"),
@@ -147,6 +154,8 @@ def reload(
         self,
         client=None,
         projection="noAcl",
+        if_etag_match=None,
+        if_etag_not_match=None,
         if_generation_match=None,
         if_generation_not_match=None,
         if_metageneration_match=None,
@@ -168,6 +177,12 @@ def reload(
                            Defaults to ``'noAcl'``. Specifies the set of
                            properties to return.
 
+        :type if_etag_match: Union[str, Set[str]]
+        :param if_etag_match: (Optional) See :ref:`using-if-etag-match`
+
+        :type if_etag_not_match: Union[str, Set[str]])
+        :param if_etag_not_match: (Optional) See :ref:`using-if-etag-not-match`
+
         :type if_generation_match: long
         :param if_generation_match:
             (Optional) See :ref:`using-if-generation-match`
@@ -205,10 +220,14 @@ def reload(
             if_metageneration_match=if_metageneration_match,
             if_metageneration_not_match=if_metageneration_not_match,
         )
+        headers = self._encryption_headers()
+        _add_etag_match_headers(
+            headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match
+        )
         api_response = client._get_resource(
             self.path,
             query_params=query_params,
-            headers=self._encryption_headers(),
+            headers=headers,
             timeout=timeout,
             retry=retry,
             _target_object=self,
@@ -384,8 +403,7 @@ def update(
 
 
 def _scalar_property(fieldname):
-    """Create a property descriptor around the :class:`_PropertyMixin` helpers.
-    """
+    """Create a property descriptor around the :class:`_PropertyMixin` helpers."""
 
     def _getter(self):
         """Scalar property getter."""
@@ -449,6 +467,24 @@ def _convert_to_timestamp(value):
     return mtime
 
 
+def _add_etag_match_headers(headers, **match_parameters):
+    """Add generation match parameters into the given parameters list.
+
+    :type headers: dict
+    :param headers: Headers dict.
+
+    :type match_parameters: dict
+    :param match_parameters: if*etag*match parameters to add.
+    """
+    for snakecase_name, header_name in _ETAG_MATCH_PARAMETERS:
+        value = match_parameters.get(snakecase_name)
+
+        if value is not None:
+            if isinstance(value, string_types):
+                value = [value]
+            headers[header_name] = ", ".join(value)
+
+
 def _add_generation_match_parameters(parameters, **match_parameters):
     """Add generation match parameters into the given parameters list.
 

google/cloud/storage/blob.py

@@ -59,6 +59,7 @@
 from google.cloud._helpers import _rfc3339_nanos_to_datetime
 from google.cloud._helpers import _to_bytes
 from google.cloud.exceptions import NotFound
+from google.cloud.storage._helpers import _add_etag_match_headers
 from google.cloud.storage._helpers import _add_generation_match_parameters
 from google.cloud.storage._helpers import _PropertyMixin
 from google.cloud.storage._helpers import _scalar_property
@@ -634,6 +635,8 @@ def generate_signed_url(
     def exists(
         self,
         client=None,
+        if_etag_match=None,
+        if_etag_not_match=None,
         if_generation_match=None,
         if_generation_not_match=None,
         if_metageneration_match=None,
@@ -651,6 +654,14 @@ def exists(
             (Optional) The client to use.  If not passed, falls back to the
             ``client`` stored on the blob's bucket.
 
+        :type if_etag_match: Union[str, Set[str]]
+        :param if_etag_match:
+            (Optional) See :ref:`using-if-etag-match`
+
+        :type if_etag_not_match: Union[str, Set[str]]
+        :param if_etag_not_match:
+            (Optional) See :ref:`using-if-etag-not-match`
+
         :type if_generation_match: long
         :param if_generation_match:
             (Optional) See :ref:`using-if-generation-match`
@@ -692,12 +703,19 @@ def exists(
             if_metageneration_match=if_metageneration_match,
             if_metageneration_not_match=if_metageneration_not_match,
         )
+
+        headers = {}
+        _add_etag_match_headers(
+            headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match
+        )
+
         try:
             # We intentionally pass `_target_object=None` since fields=name
             # would limit the local properties.
             client._get_resource(
                 self.path,
                 query_params=query_params,
+                headers=headers,
                 timeout=timeout,
                 retry=retry,
                 _target_object=None,
@@ -1002,6 +1020,8 @@ def download_to_file(
         start=None,
         end=None,
         raw_download=False,
+        if_etag_match=None,
+        if_etag_not_match=None,
         if_generation_match=None,
         if_generation_not_match=None,
         if_metageneration_match=None,
@@ -1057,6 +1077,14 @@ def download_to_file(
         :param raw_download:
             (Optional) If true, download the object without any expansion.
 
+        :type if_etag_match: Union[str, Set[str]]
+        :param if_etag_match:
+            (Optional) See :ref:`using-if-etag-match`
+
+        :type if_etag_not_match: Union[str, Set[str]]
+        :param if_etag_not_match:
+            (Optional) See :ref:`using-if-etag-not-match`
+
         :type if_generation_match: long
         :param if_generation_match:
             (Optional) See :ref:`using-if-generation-match`
@@ -1121,6 +1149,8 @@ def download_to_file(
             start=start,
             end=end,
             raw_download=raw_download,
+            if_etag_match=if_etag_match,
+            if_etag_not_match=if_etag_not_match,
             if_generation_match=if_generation_match,
             if_generation_not_match=if_generation_not_match,
             if_metageneration_match=if_metageneration_match,
@@ -1137,6 +1167,8 @@ def download_to_filename(
         start=None,
         end=None,
         raw_download=False,
+        if_etag_match=None,
+        if_etag_not_match=None,
         if_generation_match=None,
         if_generation_not_match=None,
         if_metageneration_match=None,
@@ -1168,6 +1200,14 @@ def download_to_filename(
         :param raw_download:
             (Optional) If true, download the object without any expansion.
 
+        :type if_etag_match: Union[str, Set[str]]
+        :param if_etag_match:
+            (Optional) See :ref:`using-if-etag-match`
+
+        :type if_etag_not_match: Union[str, Set[str]]
+        :param if_etag_not_match:
+            (Optional) See :ref:`using-if-etag-not-match`
+
         :type if_generation_match: long
         :param if_generation_match:
             (Optional) See :ref:`using-if-generation-match`
@@ -1233,6 +1273,8 @@ def download_to_filename(
                     start=start,
                     end=end,
                     raw_download=raw_download,
+                    if_etag_match=if_etag_match,
+                    if_etag_not_match=if_etag_not_match,
                     if_generation_match=if_generation_match,
                     if_generation_not_match=if_generation_not_match,
                     if_metageneration_match=if_metageneration_match,
@@ -1260,6 +1302,8 @@ def download_as_bytes(
         start=None,
         end=None,
         raw_download=False,
+        if_etag_match=None,
+        if_etag_not_match=None,
         if_generation_match=None,
         if_generation_not_match=None,
         if_metageneration_match=None,
@@ -1288,6 +1332,14 @@ def download_as_bytes(
         :param raw_download:
             (Optional) If true, download the object without any expansion.
 
+        :type if_etag_match: Union[str, Set[str]]
+        :param if_etag_match:
+            (Optional) See :ref:`using-if-etag-match`
+
+        :type if_etag_not_match: Union[str, Set[str]]
+        :param if_etag_not_match:
+            (Optional) See :ref:`using-if-etag-not-match`
+
         :type if_generation_match: long
         :param if_generation_match:
             (Optional) See :ref:`using-if-generation-match`
@@ -1355,6 +1407,8 @@ def download_as_bytes(
             start=start,
             end=end,
             raw_download=raw_download,
+            if_etag_match=if_etag_match,
+            if_etag_not_match=if_etag_not_match,
             if_generation_match=if_generation_match,
             if_generation_not_match=if_generation_not_match,
             if_metageneration_match=if_metageneration_match,
@@ -1371,6 +1425,8 @@ def download_as_string(
         start=None,
         end=None,
         raw_download=False,
+        if_etag_match=None,
+        if_etag_not_match=None,
         if_generation_match=None,
         if_generation_not_match=None,
         if_metageneration_match=None,
@@ -1401,6 +1457,14 @@ def download_as_string(
         :param raw_download:
             (Optional) If true, download the object without any expansion.
 
+        :type if_etag_match: Union[str, Set[str]]
+        :param if_etag_match:
+            (Optional) See :ref:`using-if-etag-match`
+
+        :type if_etag_not_match: Union[str, Set[str]]
+        :param if_etag_not_match:
+            (Optional) See :ref:`using-if-etag-not-match`
+
         :type if_generation_match: long
         :param if_generation_match:
             (Optional) See :ref:`using-if-generation-match`
@@ -1460,6 +1524,8 @@ def download_as_string(
             start=start,
             end=end,
             raw_download=raw_download,
+            if_etag_match=if_etag_match,
+            if_etag_not_match=if_etag_not_match,
             if_generation_match=if_generation_match,
             if_generation_not_match=if_generation_not_match,
             if_metageneration_match=if_metageneration_match,
@@ -1475,6 +1541,8 @@ def download_as_text(
         end=None,
         raw_download=False,
         encoding=None,
+        if_etag_match=None,
+        if_etag_not_match=None,
         if_generation_match=None,
         if_generation_not_match=None,
         if_metageneration_match=None,
@@ -1507,6 +1575,14 @@ def download_as_text(
             downloaded bytes.  Defaults to the ``charset`` param of
             attr:`content_type`, or else to "utf-8".
 
+        :type if_etag_match: Union[str, Set[str]]
+        :param if_etag_match:
+            (Optional) See :ref:`using-if-etag-match`
+
+        :type if_etag_not_match: Union[str, Set[str]]
+        :param if_etag_not_match:
+            (Optional) See :ref:`using-if-etag-not-match`
+
         :type if_generation_match: long
         :param if_generation_match:
             (Optional) See :ref:`using-if-generation-match`
@@ -1558,6 +1634,8 @@ def download_as_text(
             start=start,
             end=end,
             raw_download=raw_download,
+            if_etag_match=if_etag_match,
+            if_etag_not_match=if_etag_not_match,
             if_generation_match=if_generation_match,
             if_generation_not_match=if_generation_not_match,
             if_metageneration_match=if_metageneration_match,