Added samples for Storage, OCR, and Slack tutorial (GoogleCloudPlatform#1585)

* Added samples for Storage, OCR, and Slack tutorial

* Completed requested changes + Lint fixes (F-strings removed)

* Minor fix

* Updated the tests

* Minor fixes

* Use newer version of flask (1.0.2 instead of 0.12.2)

* Minor fix

Author: chenyumic

.gitignore

@@ -21,4 +21,5 @@ junit.xml
 credentials.dat
 .nox
 .vscode/
-*sponge_log.xml
+*sponge_log.xml
+.DS_store

functions/gcs/README.md

@@ -0,0 +1,11 @@
+<img src="https://avatars2.githubusercontent.com/u/2810941?v=3&s=96" alt="Google Cloud Platform logo" title="Google Cloud Platform" align="right" height="96" width="96"/>
+
+# Google Cloud Functions - Cloud Storage sample
+
+See:
+
+* [Cloud Functions Cloud Storage tutorial][tutorial]
+* [Cloud Functions Cloud Storage source code][code]
+
+[tutorial]: https://cloud.google.com/functions/docs/tutorials/storage
+[code]: main.py

functions/gcs/main.py

@@ -0,0 +1,34 @@
+# Copyright 2018, Google, LLC.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# [START functions_helloworld_storage_generic]
+def hello_gcs_generic(data, context):
+    """Background Cloud Function to be triggered by Cloud Storage.
+       This generic function logs relevant data when a file is changed.
+
+    Args:
+        data (dict): The Cloud Functions event payload.
+        context (google.cloud.functions.Context): Metadata of triggering event.
+    Returns:
+        None; the output is written to Stackdriver Logging
+    """
+
+    print('Event ID: {}'.format(context.event_id))
+    print('Event type: {}'.format(context.event_type))
+    print('Bucket: {}'.format(data['bucket']))
+    print('File: {}'.format(data['name']))
+    print('Metageneration: {}'.format(data['metageneration']))
+    print('Created: {}'.format(data['timeCreated']))
+    print('Updated: {}'.format(data['updated']))
+# [END functions_helloworld_storage_generic]

functions/gcs/main_test.py

@@ -0,0 +1,37 @@
+# Copyright 2018, Google, LLC.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+import main
+
+
+class TestGCFPyGCSSample(object):
+    def test_hello_gcs_generic(self, capsys):
+        event = {
+            'bucket': 'some-bucket',
+            'name': 'some-filename',
+            'metageneration': 'some-metageneration',
+            'timeCreated': '0',
+            'updated': '0'
+        }
+        context = mock.MagicMock()
+        context.event_id = 'some-id'
+        context.event_type = 'gcs-event'
+
+        main.hello_gcs_generic(event, context)
+
+        out, _ = capsys.readouterr()
+
+        assert 'some-bucket' in out
+        assert 'some-id' in out

functions/helloworld/sample_pubsub_test_system.py

@@ -58,7 +58,5 @@ def test_print_name(publisher_client):
         start_time
     ], stdout=subprocess.PIPE)
     logs = str(log_process.communicate()[0])
-    print logs
-    print start_time
     assert 'Hello, {}!'.format(name) in logs
 # [END functions_pubsub_system_test]

functions/ocr/README.md

@@ -0,0 +1,11 @@
+<img src="https://avatars2.githubusercontent.com/u/2810941?v=3&s=96" alt="Google Cloud Platform logo" title="Google Cloud Platform" align="right" height="96" width="96"/>
+
+# Google Cloud Functions - OCR (Optical Character Recognition) sample
+
+See:
+
+* [Cloud Functions OCR tutorial][tutorial]
+* [Cloud Functions OCR sample source code][code]
+
+[tutorial]: https://cloud.google.com/functions/docs/tutorials/ocr
+[code]: app/main.py

functions/ocr/app/config.json

@@ -0,0 +1,7 @@
+{
+  "RESULT_TOPIC": "[YOUR_RESULT_TOPIC_NAME]",
+  "RESULT_BUCKET": "[YOUR_TEXT_BUCKET_NAME]",
+  "TRANSLATE_TOPIC": "[YOUR_TRANSLATE_TOPIC_NAME]",
+  "TRANSLATE": true,
+  "TO_LANG": ["en", "fr", "es", "ja", "ru"]
+}

functions/ocr/app/main.py

@@ -0,0 +1,161 @@
+# Copyright 2018, Google, LLC.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START functions_ocr_setup]
+import base64
+import json
+import os
+
+from google.cloud import pubsub_v1
+from google.cloud import storage
+from google.cloud import translate
+from google.cloud import vision
+
+vision_client = vision.ImageAnnotatorClient()
+translate_client = translate.Client()
+publisher = pubsub_v1.PublisherClient()
+storage_client = storage.Client()
+
+project_id = os.environ['GCLOUD_PROJECT']
+
+with open('config.json') as f:
+    data = f.read()
+config = json.loads(data)
+# [END functions_ocr_setup]
+
+
+# [START functions_ocr_detect]
+def detect_text(bucket, filename):
+    print('Looking for text in image {}'.format(filename))
+
+    futures = []
+
+    text_detection_response = vision_client.text_detection({
+        'source': {'image_uri': 'gs://{}/{}'.format(bucket, filename)}
+    })
+    annotations = text_detection_response.text_annotations
+    if len(annotations) > 0:
+        text = annotations[0].description
+    else:
+        text = ''
+    print('Extracted text {} from image ({} chars).'.format(text, len(text)))
+
+    detect_language_response = translate_client.detect_language(text)
+    src_lang = detect_language_response['language']
+    print('Detected language {} for text {}.'.format(src_lang, text))
+
+    # Submit a message to the bus for each target language
+    for target_lang in config.get('TO_LANG', []):
+        topic_name = config['TRANSLATE_TOPIC']
+        if src_lang == target_lang:
+            topic_name = config['RESULT_TOPIC']
+        message = {
+            'text': text,
+            'filename': filename,
+            'target_lang': target_lang,
+            'src_lang': src_lang
+        }
+        message_data = json.dumps(message).encode('utf-8')
+        topic_path = publisher.topic_path(project_id, topic_name)
+        future = publisher.publish(topic_path, data=message_data)
+        futures.append(future)
+    for future in futures:
+        future.result()
+# [END functions_ocr_detect]
+
+
+# [START message_validatation_helper]
+def validate_message(message, param):
+    var = message.get(param)
+    if not var:
+        raise ValueError('{} is not provided. Make sure you have \
+                          property {} in the request'.format(param, param))
+    return var
+# [END message_validatation_helper]
+
+
+# [START functions_ocr_process]
+def process_image(file, context):
+    """Cloud Function triggered by Cloud Storage when a file is changed.
+    Args:
+        file (dict): Metadata of the changed file, provided by the triggering
+                                 Cloud Storage event.
+        context (google.cloud.functions.Context): Metadata of triggering event.
+    Returns:
+        None; the output is written to stdout and Stackdriver Logging
+    """
+    bucket = validate_message(file, 'bucket')
+    name = validate_message(file, 'name')
+
+    detect_text(bucket, name)
+
+    print('File {} processed.'.format(file['name']))
+# [END functions_ocr_process]
+
+
+# [START functions_ocr_translate]
+def translate_text(event, context):
+    if event.get('data'):
+        message_data = base64.b64decode(event['data']).decode('utf-8')
+        message = json.loads(message_data)
+    else:
+        raise ValueError('Data sector is missing in the Pub/Sub message.')
+
+    text = validate_message(message, 'text')
+    filename = validate_message(message, 'filename')
+    target_lang = validate_message(message, 'target_lang')
+    src_lang = validate_message(message, 'src_lang')
+
+    print('Translating text into {}.'.format(target_lang))
+    translated_text = translate_client.translate(text,
+                                                 target_language=target_lang,
+                                                 source_language=src_lang)
+    topic_name = config['RESULT_TOPIC']
+    message = {
+        'text': translated_text['translatedText'],
+        'filename': filename,
+        'lang': target_lang,
+    }
+    message_data = json.dumps(message).encode('utf-8')
+    topic_path = publisher.topic_path(project_id, topic_name)
+    future = publisher.publish(topic_path, data=message_data)
+    future.result()
+# [END functions_ocr_translate]
+
+
+# [START functions_ocr_save]
+def save_result(event, context):
+    if event.get('data'):
+        message_data = base64.b64decode(event['data']).decode('utf-8')
+        message = json.loads(message_data)
+    else:
+        raise ValueError('Data sector is missing in the Pub/Sub message.')
+
+    text = validate_message(message, 'text')
+    filename = validate_message(message, 'filename')
+    lang = validate_message(message, 'lang')
+
+    print('Received request to save file {}.'.format(filename))
+
+    bucket_name = config['RESULT_BUCKET']
+    result_filename = '{}_{}.txt'.format(filename, lang)
+    bucket = storage_client.get_bucket(bucket_name)
+    blob = bucket.blob(result_filename)
+
+    print('Saving result to {} in bucket {}.'.format(result_filename,
+                                                     bucket_name))
+
+    blob.upload_from_string(text)
+
+    print('File saved.')
+# [END functions_ocr_save]

functions/ocr/app/main_test.py

@@ -0,0 +1,89 @@
+# Copyright 2018, Google, LLC.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import concurrent.futures
+import json
+
+import mock
+
+import main
+
+
+class TestGCFPyOCRSample():
+    @mock.patch.object(main, 'publisher')
+    @mock.patch.object(main, 'translate_client')
+    @mock.patch.object(main, 'vision_client')
+    def test_detect_text(self, mock_vision_client, mock_translate_client,
+                         mock_publisher):
+        mock_annotation = mock.MagicMock()
+        mock_annotation.description = 'sample text'
+        mock_annotations = mock.MagicMock()
+        mock_annotations.text_annotations = [mock_annotation]
+        mock_vision_client.text_detection.return_value = mock_annotations
+
+        mock_translate_client.detect_language.return_value = {'language': 'en'}
+
+        mock_future = concurrent.futures.Future()
+        mock_future.set_result(True)
+        mock_publisher.publish.return_value = mock_future
+
+        main.detect_text('sample-bucket', 'sample-file')
+
+    @mock.patch.object(main, 'detect_text')
+    def test_process_image(self, m):
+        m.return_value = None
+        event = {
+            'bucket': 'sample-bucket',
+            'name': 'sample-file'
+        }
+        context = {}
+        main.process_image(event, context)
+
+    @mock.patch.object(main, 'publisher')
+    @mock.patch.object(main, 'translate_client')
+    def test_translate_text(self, mock_translate_client, mock_publisher):
+        mock_translate_client.translate.return_value = {'translatedText': ''}
+
+        mock_future = concurrent.futures.Future()
+        mock_future.set_result(True)
+        mock_publisher.publish.return_value = mock_future
+
+        data = base64.b64encode(json.dumps({
+            'text': 'menu',
+            'filename': 'sample-file',
+            'target_lang': 'es',
+            'src_lang': 'en'
+        }).encode('utf-8'))
+        event = {
+            'data': data
+        }
+        context = {}
+        main.translate_text(event, context)
+
+    @mock.patch.object(main, 'storage_client')
+    def test_save_result(self, m):
+        bucket = m.bucket.return_value
+        file = bucket.file.return_value
+        file.save.return_value = None
+
+        data = base64.b64encode(json.dumps({
+            'text': 'menu',
+            'filename': 'sample-file',
+            'lang': 'fr',
+        }).encode('utf-8'))
+        event = {
+            'data': data
+        }
+        context = {}
+        main.save_result(event, context)

functions/ocr/app/requirements.txt

@@ -0,0 +1,4 @@
+google-cloud-pubsub==0.35.4
+google-cloud-storage==1.10.0
+google-cloud-translate==1.3.1
+google-cloud-vision==0.32.0