Merge branch 'main' into production

.gitlab-ci.yml

@@ -128,7 +128,7 @@ build_docker:
   script:
     - API_DOMAIN=$(cat $API_DOMAIN_PATH)
     - terraform/environments/scripts/wait-for-ssl.sh "https://${API_DOMAIN}"
-    - BASE_URL="https://${API_DOMAIN}/graphql" /tmp/k6 run smoketest/main.js
+    - BASE_URL="https://${API_DOMAIN}/graphql" k6 run smoketest/main.js
     # TODO should/could we roll back the service to the last working revision on test failure?
 
 review_deploy:
@@ -177,7 +177,7 @@ review_destroy:
     - terraform/environments/scripts/destroy-env.sh $CI_ENVIRONMENT_SLUG
   # can't use rules here: https://gitlab.com/gitlab-org/gitlab/-/issues/34077
   when: manual
-  allow_failure: false
+  allow_failure: true
   except:
     - main
     - production

openbook/settings/__init__.py

@@ -231,6 +231,7 @@ CACHES = {
         "BACKEND": CACHE_BACKENDS["redis"],
         "LOCATION": REDIS_DEFAULT_CACHE_LOCATION,
         "OPTIONS": {
+            "PARSER_CLASS": "redis.connection._HiredisParser",
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "REDIS_CLIENT_KWARGS": {"health_check_interval": 30},
         },
@@ -241,6 +242,7 @@ CACHES = {
         "BACKEND": CACHE_BACKENDS["redis"],
         "LOCATION": REDIS_USERBLOCK_CACHE_LOCATION,
         "OPTIONS": {
+            "PARSER_CLASS": "redis.connection._HiredisParser",
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "REDIS_CLIENT_KWARGS": {"health_check_interval": 30},
         },
@@ -251,19 +253,24 @@ CACHES = {
         "BACKEND": CACHE_BACKENDS["redis"],
         "LOCATION": REDIS_UNIFIED_API_RESPONSE_CACHE_LOCATION,
         "OPTIONS": {
+            "PARSER_CLASS": "redis.connection._HiredisParser",
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "SERIALIZER": "django_redis.serializers.json.JSONSerializer",
             "REDIS_CLIENT_KWARGS": {"health_check_interval": 30},
+            "SOCKET_TIMEOUT": 5,
+            "SOCKET_CONNECT_TIMEOUT": 5,
         },
         "KEY_PREFIX": "response-cache",
         "KEY_FUNCTION": "openbook.utils.unified_api_response_cache_key_function",
         "REVERSE_KEY_FUNCTION": "openbook.utils.unified_api_response_cache_reverse_key",
         "TIMEOUT": 60 * 60,
+        "CONNECTION_POOL_CLASS_KWARGS": {"retry_on_timeout": True},
     },
     RQ_DEFAULT_JOBS: {
         "BACKEND": CACHE_BACKENDS["redis"],
         "LOCATION": REDIS_RQ_DEFAULT_JOBS_CACHE_LOCATION,
         "OPTIONS": {
+            "PARSER_CLASS": "redis.connection._HiredisParser",
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "REDIS_CLIENT_KWARGS": {"health_check_interval": 30},
         },
@@ -273,6 +280,7 @@ CACHES = {
         "BACKEND": CACHE_BACKENDS["redis"],
         "LOCATION": REDIS_RQ_HIGH_JOBS_CACHE_LOCATION,
         "OPTIONS": {
+            "PARSER_CLASS": "redis.connection._HiredisParser",
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "REDIS_CLIENT_KWARGS": {"health_check_interval": 30},
         },
@@ -282,6 +290,7 @@ CACHES = {
         "BACKEND": CACHE_BACKENDS["redis"],
         "LOCATION": REDIS_RQ_LOW_JOBS_CACHE_LOCATION,
         "OPTIONS": {
+            "PARSER_CLASS": "redis.connection._HiredisParser",
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
             "REDIS_CLIENT_KWARGS": {"health_check_interval": 30},
         },

openbook_notifications/notifications.py

 from __future__ import annotations
 
+import json
 import uuid
 from enum import Enum
 from typing import List, Optional
@@ -10,7 +11,6 @@ from requests import Session
 from requests.adapters import HTTPAdapter, Retry
 from requests.exceptions import ConnectionError
 
-from openbook_notifications.helpers import flatten_payload
 from openbook_notifications.payloads import (
     PostCommentAuthorPayload,
     PostCommentMentionPayload,
@@ -74,9 +74,16 @@ class Notification:
         self.session = session
         self.transaction_id = transaction_id
 
+        # Android notifications only allow for key/value pairs in the data field to be of type string, not objects/JSON
+        # (see https://firebase.google.com/docs/reference/fcm/rest/v1/projects.messages#androidconfig).
+        # That is why we serialize the nested data object to a string and deserialize back to JSON it in the mobile client.
+        android_payload = {k: v for k, v in payload.items() if not isinstance(v, dict)}
+        android_payload["data"] = json.dumps(payload.get("data"))
+        # iOS allows to use full JSON-serializable objects as payload
+        # (see https://firebase.google.com/docs/reference/fcm/rest/v1/projects.messages#apnsconfig)
         self.overrides = {
             "type": "data",  # data notification
-            "fcm": {"type": "notification", "data": flatten_payload(payload)},
+            "fcm": {"type": "notification", "data": android_payload},
             "apns": {"payload": payload},
         }
 

openbook_notifications/tests/test_notifications.py

@@ -57,16 +57,20 @@ def mock_novu_api():
 class NotificationTestCase(TestCase):
     @classmethod
     def setUpTestData(cls):
+        cls.payload = {"data": {"key": "value", "nested": {"nestedKey": "nestedValue"}}}
         cls.notification_instance = Notification(
             workflow="test-workflow",
             recipients=["user-id"],
-            payload={"data": {"key": "value"}},
+            payload=cls.payload,
             transaction_id="test-transaction-id",
         )
         cls.overrides = {
             "type": "data",
-            "fcm": {"type": "notification", "data": {"key": "value"}},
-            "apns": {"payload": {"data": {"key": "value"}}},
+            "fcm": {
+                "type": "notification",
+                "data": {"data": '{"key": "value", "nested": {"nestedKey": "nestedValue"}}'},
+            },
+            "apns": {"payload": cls.payload},
         }
 
     @patch.object(EventApi, "trigger")
@@ -77,7 +81,7 @@ class NotificationTestCase(TestCase):
         mock_event_api_trigger.assert_called_with(
             name="test-workflow",
             recipients=["user-id"],
-            payload={"data": {"key": "value"}},
+            payload=self.payload,
             overrides=self.overrides,
             transaction_id="test-transaction-id",
         )

renovate.json

@@ -36,6 +36,13 @@
       ],
       "automerge": true
     },
+    {
+      "matchDepNames": [
+        "boto3",
+        "botocore"
+      ],
+      "groupName": "boto3"
+    },
     {
       "matchDepTypes": [
         "devDependencies"

requirements.txt

@@ -22,12 +22,12 @@ bandit~=1.7.9
 beautifulsoup4~=4.12.3
 black~=24.10.0
 blurhash-python~=1.2.2
-boto3~=1.34.154
-botocore~=1.34.154
-cachetools~=5.4.0
+boto3~=1.35.46
+botocore~=1.35.46
+cachetools~=5.5.0
 certifi~=2024.8.30
 cffi~=1.17.0
-charset-normalizer~=3.3.2
+charset-normalizer~=3.4.0
 click~=8.1.3
 colorama~=0.4.6
 colorlog~=6.8.2
@@ -38,7 +38,7 @@ Django~=5.0.7
 django-admin-rangefilter~=0.13.1
 django-appconf~=1.0.6
 django-cacheops~=7.0.2
-django-cors-headers~=4.4.0
+django-cors-headers~=4.5.0
 django-cursor-pagination~=0.3.0
 django-debug-toolbar~=4.4.6
 django-extensions~=3.2.3
@@ -58,22 +58,23 @@ djangorestframework-camel-case~=1.4.2
 dparse~=0.6.4b0
 execnet~=2.1.1
 Faker~=12.0.1 # mixer 7.2.2 depends on Faker<12.1 and >=5.4.0
-filelock~=3.15.4
-frozenlist~=1.4.1
+filelock~=3.16.1
+frozenlist~=1.5.0
 funcy~=2.0
 gitdb~=4.0.11
 GitPython~=3.1.43
-google-api-core~=2.19.1
-google-auth~=2.33.0
-google-cloud-pubsub~=2.23.0
+google-api-core~=2.21.0
+google-auth~=2.35.0
+google-cloud-pubsub~=2.26.1
 google-cloud-webrisk~=1.14.5
-googleapis-common-protos~=1.63.0
+googleapis-common-protos~=1.65.0
 graphql-core~=3.2.3
 grpc-google-iam-v1~=0.13.0
-grpcio~=1.65.4
+grpcio~=1.67.0
 grpcio-status~=1.62.1
 h11~=0.14.0
 halo~=0.0.31
+hiredis~=3.0.0
 httpcore~=1.0.5
 httpx~=0.27.0
 icalendar~=5.0.13
@@ -86,10 +87,10 @@ langdetect~=1.0.9
 log-symbols~=0.0.14
 markdown-it-py~=3.0.0
 MarkupSafe~=2.1.5
-marshmallow~=3.21.3
+marshmallow~=3.23.0
 mdurl~=0.1.2
 mixer~=7.2.2
-multidict~=6.0.5
+multidict~=6.1.0
 mypy-extensions~=1.0.0
 nose~=1.3.7
 nose-exclude~=0.5.0
@@ -97,14 +98,14 @@ novu~=1.14.0
 packaging~=24.1
 pathspec~=0.12.1
 pathtools~=0.1.2
-pbr~=6.0.0
+pbr~=6.1.0
 pilkit~=3.0
 pillow~=10.4.0
 pinocchio~=0.4.3
-platformdirs~=4.2.0
+platformdirs~=4.3.6
 pluggy~=1.5.0
 posthog==3.7.0
-proto-plus~=1.24.0
+proto-plus~=1.25.0
 protobuf~=4.25.3
 psycopg~=3.2.1
 psycopg-binary~=3.2.1
@@ -115,11 +116,11 @@ pydantic~=2.8.2
 pydantic_core~=2.20.1
 Pygments~=2.18.0
 PyJWT~=2.9.0
-pyparsing~=3.1.2
+pyparsing~=3.2.0
 pytest~=8.3.2
-pytest-asyncio~=0.23.8
+pytest-asyncio~=0.24.0
 pytest-cov~=5.0.0
-pytest-django~=4.8.0
+pytest-django~=4.9.0
 pytest-xdist~=3.6.1
 python-benedict~=0.33.2
 python-dateutil~=2.9.0.post0