diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 4a5d37796d8189bb1509903bc9cd11bc45ca3215..0000000000000000000000000000000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,149 +0,0 @@ -# Python CircleCI 2.0 configuration file -# -# Check https://circleci.com/docs/2.0/language-python/ for more details -# -version: 2 -jobs: - build: - docker: - # specify the version you desire here - # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` - - image: circleci/python:3.10.1 - - # Specify service dependencies here if necessary - # CircleCI maintains a library of pre-built images - # documented at https://circleci.com/docs/2.0/circleci-images/ - # - image: circleci/postgres:9.4 - - working_directory: ~/repo - - steps: - - checkout - # Download and cache dependencies - - restore_cache: - keys: - - $CACHE_VERSION-dependencies-{{ checksum "requirements.txt" }} - # fallback to using the latest cache if no exact match is found - - $CACHE_VERSION-dependencies- - - - run: - name: install dependencies - command: | - sudo apt-get -y -qq update - sudo apt-get install ffmpeg redis-server libmagic-dev - sudo /etc/init.d/redis-server start - python3 -m venv venv - . venv/bin/activate - pip install --upgrade pip - pip install -r requirements.txt --exists-action s - - - save_cache: - paths: - - ./venv - key: $CACHE_VERSION-pip-dependencies-{{ checksum "requirements.txt" }} - - - run: - name: Setup Code Climate test-reporter - command: | - # download test reporter as a static binary - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter - chmod +x ./cc-test-reporter - - run: - name: run tests - command: | - . venv/bin/activate - # notify Code Climate of a pending test report using `before-build` - ./cc-test-reporter before-build - python manage.py test - # upload test report to Code Climate using `after-build` - ./cc-test-reporter after-build --exit-code $? - # static code analysis for basic secure coding practices - - run: - name: run bandit - command: | - . venv/bin/activate - bandit -r . - - # check for known vulns in python packages - - run: - name: run safety - command: | - . venv/bin/activate - safety check - - deploy: - working_directory: ~/repo - docker: - - image: circleci/python:3.10.1 - steps: - - checkout - - run: - name: Install utils dependencies - command: | - python3 -m venv venv - . venv/bin/activate - pip install jinja2 - - run: - name: Installing deployment dependencies - working_directory: / - command: | - sudo apt-get -y -qq update - sudo apt-get install python-pip python-dev build-essential - sudo pip install awsebcli --upgrade - - run: - name: Prepare environment variables for creating admin config - command: | - if [ $CIRCLE_BRANCH == 'master' ]; then - echo "export LB_NAME=$LB_NAME_MASTER" >> $BASH_ENV - echo "export SAUTH_SERVER_NAME=$SAUTH_SERVER_NAME_MASTER" >> $BASH_ENV - elif [ $CIRCLE_BRANCH == 'develop' ]; then - echo "export LB_NAME=$LB_NAME_DEVELOP" >> $BASH_ENV - echo "export SAUTH_SERVER_NAME=$SAUTH_SERVER_NAME_DEVELOP" >> $BASH_ENV - else - echo "export LB_NAME=lb-not-existing.localhost" >> $BASH_ENV - echo "export SAUTH_SERVER_NAME=sauth-not-existing.localhost" >> $BASH_ENV - fi - - run: - name: Make EB Config - command: | - . venv/bin/activate - python utils/make_eb_config.py --name=$APPLICATION_NAME --region=$AWS_DEFAULT_REGION - - run: - name: Create magic header authentication for protected environments - command: | - if [ $CIRCLE_BRANCH == 'develop' ]; then - . venv/bin/activate - python utils/make_magic_header.py --name=$DEVELOP_MAGIC_HEADER_NAME --value=$DEVELOP_MAGIC_HEADER_VALUE - fi - if [ $CIRCLE_BRANCH == 'master' ]; then - . venv/bin/activate - python utils/make_magic_header_admin.py --name=$MASTER_MAGIC_HEADER_ADMIN_NAME --value=$MASTER_MAGIC_HEADER_ADMIN_VALUE - fi - - run: - name: Commit Dynamic EB Extensions - command: | - git config --global user.email "circle@ci.com" - git config --global user.name "CIRCLECI" - git add .ebextensions/ - git commit -m "EB Uploads only committed files, therefore this" - - run: - name: Deploy to EB - command: | - if [ $CIRCLE_BRANCH == 'master' ]; then - eb deploy master-$MASTER_APPLICATION_NAME - elif [ $CIRCLE_BRANCH == 'develop' ]; then - eb deploy develop-$DEVELOP_APPLICATION_NAME - else - echo "Not deploying as its not master nor develop branches" - fi -workflows: - version: 2 - build: - jobs: - - build - - deploy: - filters: - branches: - only: - - master - - develop diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdb8cf803c2f5129b43387b48b3dd2eb8f234cf4..e9539f202971afc9e415fd865048befb58bd6b63 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,9 +20,7 @@ stages: default: before_script: - set -eu - # env -0 | sort -z | tr '\0' '\n': Sort env output alphabetically, keeping multiline variables intact - # egrep: Remove sensitive information from the output of env - #- env -0 | sort -z | tr '\0' '\n' | egrep -ve '^(DOCKER_AUTH_CONFIG|GOOGLE_APPLICATION_CREDENTIALS)=.*' + # DANGER don't use `set -x` or print the environment via e.g. `env` in pipeline runs, this might leak credentials (has leaked them) interruptible: true tags: - 1cpu-4gb # build on smaller machine @@ -153,7 +151,7 @@ review_smoketest: review_destroy: stage: destroy image: - name: 'europe-north1-docker.pkg.dev/holi-shared/docker-hub-remote/hashicorp/terraform:1.11.1' + name: 'europe-north1-docker.pkg.dev/holi-shared/docker-hub-remote/hashicorp/terraform:1.11.2' # default entrypoint is terraform command, but we want to run shell scripts entrypoint: ['/bin/sh', '-c'] variables: @@ -194,13 +192,15 @@ staging_deploy: url: https://staging.social.apis.holi.social variables: ENVIRONMENT_ID: staging - only: - - main + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: on_success staging_smoketest: extends: .smoketest - only: - - main + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: on_success staging_trigger_unified-api_redeployment: stage: downstream @@ -210,8 +210,9 @@ staging_trigger_unified-api_redeployment: forward: yaml_variables: false pipeline_variables: false - only: - - main + rules: + - if: $CI_COMMIT_BRANCH == "main" + when: on_success resource_group: unified-api-staging production_deploy: @@ -223,13 +224,15 @@ production_deploy: url: https://production.social.apis.holi.social variables: ENVIRONMENT_ID: production - only: - - production + rules: + - if: $CI_COMMIT_BRANCH == "production" + when: on_success production_smoketest: extends: .smoketest - only: - - production + rules: + - if: $CI_COMMIT_BRANCH == "production" + when: on_success production_trigger_unified-api_redeployment: stage: downstream @@ -239,6 +242,7 @@ production_trigger_unified-api_redeployment: forward: yaml_variables: false pipeline_variables: false - only: - - production + rules: + - if: $CI_COMMIT_BRANCH == "production" + when: on_success resource_group: unified-api-production diff --git a/.terraform-version b/.terraform-version index 720c7384c6195b916c795feaddb08bbe023d183c..ca7176690dd6f501842f3ef4b70bb32118edb489 100644 --- a/.terraform-version +++ b/.terraform-version @@ -1 +1 @@ -1.11.1 +1.11.2 diff --git a/comments/models/comment.py b/comments/models/comment.py index 106b57cda0384aede9159682cef0ff8f34327317..717b2a70b1502f2909b477cca48957b51c77bdea 100644 --- a/comments/models/comment.py +++ b/comments/models/comment.py @@ -35,7 +35,6 @@ from openbook_common.utils.model_loaders import ( ) from openbook_moderation.models import ModeratedObject from openbook_notifications.services import NotificationsService - from .comment_reaction import CommentReaction from .enabled_commentable_type import EnabledCommentableType @@ -240,7 +239,7 @@ class Comment(ModelWithUUID): username = username.lower() if username not in existing_mention_usernames: try: - user = User.objects.only("id", "username").get(username__iexact=username) + user = User.objects.get(username__iexact=username) user_can_see_commentable = self.commentable.can_be_viewed(user) user_is_commenter = user.pk == self.commenter_id diff --git a/feed_posts/admin.py b/feed_posts/admin.py index 27c266e4d52616708be70358e2b32499962e92ee..9805094d5b31c48aebb2a1fd2d27cb7e3ebc2228 100644 --- a/feed_posts/admin.py +++ b/feed_posts/admin.py @@ -29,6 +29,8 @@ class FeedPostAdminModel(admin.ModelAdmin): search_fields = ("title", "creator") + exclude = ["geolocation_geojson"] + def display_topics(self, obj): return ", ".join([topic.title for topic in obj.topics.all()]) diff --git a/feed_posts/migrations/0020_feedpost_geolocation_geojson.py b/feed_posts/migrations/0020_feedpost_geolocation_geojson.py new file mode 100644 index 0000000000000000000000000000000000000000..d8023085d0d870f360363c507d3db4cea2ead478 --- /dev/null +++ b/feed_posts/migrations/0020_feedpost_geolocation_geojson.py @@ -0,0 +1,46 @@ +# Generated by Django 5.0.13 on 2025-03-20 04:42 + +from django.db import migrations, models + +from openbook_common.utils.model_loaders import get_feed_post_model + + +def get_table_name() -> str: + FeedPostModel = get_feed_post_model() + return FeedPostModel._meta.db_table + + +def run_migration(apps, schema_editor): + schema_editor.execute(f""" + UPDATE {get_table_name()} + SET geolocation_geojson = jsonb_build_object( + 'type', 'Point', + 'coordinates', ARRAY[ + ST_X(geolocation::geometry), + ST_Y(geolocation::geometry) + ] + ) + WHERE geolocation IS NOT NULL; + """) + + +def reverse_migration(apps, schema_editor): + schema_editor.execute(f""" + UPDATE {get_table_name()} + SET geolocation_geojson = NULL; + """) + + +class Migration(migrations.Migration): + dependencies = [ + ("feed_posts", "0019_feedpost_feed_posts__topics__9ffb13_gin"), + ] + + operations = [ + migrations.AddField( + model_name="feedpost", + name="geolocation_geojson", + field=models.JSONField(blank=True, null=True), + ), + migrations.RunPython(run_migration, reverse_migration), + ] diff --git a/feed_posts/models.py b/feed_posts/models.py index 8621461375e0bc0046684fc4e166773dd8c35505..cd029f9c387389cbbd9318c6b73712306a5ce38c 100644 --- a/feed_posts/models.py +++ b/feed_posts/models.py @@ -20,7 +20,7 @@ from feed_posts.helpers import upload_to_feed_post_images_directory, upload_to_n from openbook import settings from openbook_auth.models import User, UserRole from openbook_common.enums import ReactionType -from openbook_common.helpers import ExifRotate, generate_blurhash +from openbook_common.helpers import ExifRotate, generate_blurhash, geojson_from_geolocation from openbook_common.models import ModelWithUUID, LinkPreview from openbook_common.types import LanguageCode from openbook_common.utils.helpers import delete_file_field @@ -84,6 +84,8 @@ class FeedPost(CommentNotificationMixin, ModelWithUUID): ) geolocation = models.PointField(blank=True, null=True, spatial_index=True) + # also store as GeoJSON as downstream processes (Google Datastream) can't understand Postgis binary format + geolocation_geojson = models.JSONField(blank=True, null=True) priority_duration = models.PositiveIntegerField( default=0, @@ -110,6 +112,10 @@ class FeedPost(CommentNotificationMixin, ModelWithUUID): base_slug = slugify(self.title) suffix = str(uuid.uuid4())[:8] self.slug = f"{base_slug}-{suffix}" + if self.geolocation: + self.geolocation_geojson = geojson_from_geolocation(self.geolocation) + else: + self.geolocation_geojson = None super().save(*args, **kwargs) diff --git a/openbook/settings/__init__.py b/openbook/settings/__init__.py index c96871d447ce9aa12cfb92276fdcee9eadf50355..0576b97c97345f9266027d242d7d98287991642b 100644 --- a/openbook/settings/__init__.py +++ b/openbook/settings/__init__.py @@ -600,9 +600,14 @@ AWS_STATIC_LOCATION = "static" AWS_PRIVATE_MEDIA_LOCATION = os.environ.get("AWS_PRIVATE_MEDIA_LOCATION") AWS_DEFAULT_ACL = None -# TODO Think about what to do for storage backend - -DEFAULT_FILE_STORAGE = "openbook.storage_imagekit_io.ImagekitIoStorage" +STORAGES = { + "default": { + "BACKEND": "openbook.storage_imagekit_io.ImagekitIoStorage", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, +} # ONE SIGNAL ONE_SIGNAL_APP_ID = os.environ.get("ONE_SIGNAL_APP_ID") diff --git a/openbook/settings/testing.py b/openbook/settings/testing.py index c17040782867ad3ff15b0209db0f832505b7dba6..8e76e5abe3248496def37271aaf4f509b11af4d5 100644 --- a/openbook/settings/testing.py +++ b/openbook/settings/testing.py @@ -1,17 +1,16 @@ -import os from . import * # noqa: F403 ENVIRONMENT = EnvironmentChecker.TEST_VALUE # noqa: F405 TESTING = True -for queueConfig in RQ_QUEUES.values(): # noqa: F405, N816 - queueConfig["ASYNC"] = False +for queue_config in RQ_QUEUES.values(): # noqa: F405, N816 + queue_config["ASYNC"] = False -TEST_DB_NAME = os.environ.get("TEST_DB_NAME", "okuna_test_db") -TEST_DB_USERNAME = os.environ.get("TEST_DB_USERNAME", "okuna_test_user") -TEST_DB_PASSWORD = os.environ.get("TEST_DB_PASSWORD", "okuna_test_password") -TEST_DB_HOSTNAME = os.environ.get("TEST_DB_HOSTNAME", "postgres") -TEST_DB_PORT = os.environ.get("TEST_DB_PORT") +TEST_DB_NAME = os.environ.get("TEST_DB_NAME", "okuna_test_db") # noqa: F405 +TEST_DB_USERNAME = os.environ.get("TEST_DB_USERNAME", "okuna_test_user") # noqa: F405 +TEST_DB_PASSWORD = os.environ.get("TEST_DB_PASSWORD", "okuna_test_password") # noqa: F405 +TEST_DB_HOSTNAME = os.environ.get("TEST_DB_HOSTNAME", "postgres") # noqa: F405 +TEST_DB_PORT = os.environ.get("TEST_DB_PORT") # noqa: F405 DATABASES = { "default": { @@ -29,7 +28,27 @@ MIN_UNIQUE_TOP_POST_REACTIONS_COUNT = 1 MIN_UNIQUE_TOP_POST_COMMENTS_COUNT = 1 MIN_UNIQUE_TRENDING_POST_REACTIONS_COUNT = 1 -DEFAULT_FILE_STORAGE = "django.core.files.storage.FileSystemStorage" +STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, +} APPLE_APP_STORE_DEFAULT_URL = "itms-apps://apps.apple.com/app/HOLITEST" GOOGLE_PLAY_STORE_DEFAULT_URL = "market://details?id=foundation.holistic.HOLITEST" + + +# speed up pytest by not running migrations +# see https://stackoverflow.com/questions/36487961/django-unit-testing-taking-a-very-long-time-to-create-test-database +class DisableMigrations(object): + def __contains__(self, item): + return True + + def __getitem__(self, item): + return None + + +MIGRATION_MODULES = DisableMigrations() diff --git a/openbook_appointments/admin.py b/openbook_appointments/admin.py index ab7cfbfe6b7131c14b0bdc192b7a08204eaa3c34..6a9a4a61140d4fc12c44c231232f3b1c4f01193c 100644 --- a/openbook_appointments/admin.py +++ b/openbook_appointments/admin.py @@ -38,6 +38,8 @@ class AppointmentAdminModel(admin.ModelAdmin): "space_name", ) + exclude = ["geolocation_geojson"] + def delete_queryset(self, request: HttpRequest, queryset: QuerySet) -> None: queryset.filter(is_deleted=False).update(is_deleted=True, deleted_at=timezone.now()) diff --git a/openbook_appointments/migrations/0010_appointment_geolocation_geojson_and_more.py b/openbook_appointments/migrations/0010_appointment_geolocation_geojson_and_more.py new file mode 100644 index 0000000000000000000000000000000000000000..250cbffafa6ab25af29574b0d157aec55c3852d7 --- /dev/null +++ b/openbook_appointments/migrations/0010_appointment_geolocation_geojson_and_more.py @@ -0,0 +1,67 @@ +# Generated by Django 5.0.13 on 2025-03-20 04:42 + +from django.db import migrations, models +from django.apps import apps + +from openbook_common.utils.model_loaders import get_appointment_model + + +def appointments_table_name() -> str: + AppointmentsModel = apps.get_model("openbook_appointments.Appointment") + return AppointmentsModel._meta.db_table + + +def location_details_table_name() -> str: + LocationDetailsModel = apps.get_model("openbook_appointments.LocationDetails") + return LocationDetailsModel._meta.db_table + + +def migration_sql(table_name) -> str: + return f""" + UPDATE {table_name} + SET geolocation_geojson = jsonb_build_object( + 'type', 'Point', + 'coordinates', ARRAY[ + ST_X(geolocation::geometry), + ST_Y(geolocation::geometry) + ] + ) + WHERE geolocation IS NOT NULL; + """ + + +def reverse_migration_sql(table_name) -> str: + return f""" + UPDATE {table_name} + SET geolocation_geojson = NULL; + """ + + +def run_migration(apps, schema_editor): + schema_editor.execute(migration_sql(appointments_table_name())) + schema_editor.execute(migration_sql(location_details_table_name())) + + +def reverse_migration(apps, schema_editor): + schema_editor.execute(reverse_migration_sql(location_details_table_name())) + schema_editor.execute(reverse_migration_sql(appointments_table_name())) + + +class Migration(migrations.Migration): + dependencies = [ + ("openbook_appointments", "0009_appointment_deleted_at_appointment_is_deleted"), + ] + + operations = [ + migrations.AddField( + model_name="appointment", + name="geolocation_geojson", + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name="locationdetails", + name="geolocation_geojson", + field=models.JSONField(blank=True, null=True), + ), + migrations.RunPython(run_migration, reverse_migration), + ] diff --git a/openbook_appointments/models.py b/openbook_appointments/models.py index 5604af1d985081d047d816926a0a20bf7b482770..4a6fac9d6d5d5ddc6b8f28e615a360cefaa187f6 100644 --- a/openbook_appointments/models.py +++ b/openbook_appointments/models.py @@ -14,7 +14,7 @@ from openbook import settings from openbook_appointments.enums import PermissionType from openbook_appointments.helpers import upload_to_space_appointment_thumbnail_directory from openbook_auth.models import User -from openbook_common.helpers import ExifRotate, generate_blurhash +from openbook_common.helpers import ExifRotate, generate_blurhash, geojson_from_geolocation from openbook_common.models import ModelWithUUID from openbook_common.utils.helpers import delete_file_field from openbook_communities.models import Community @@ -44,6 +44,8 @@ class Appointment(ModelWithUUID): ) geolocation = models.PointField(blank=True, null=True, spatial_index=True) + # also store as GeoJSON as downstream processes (Google Datastream) can't understand Postgis binary format + geolocation_geojson = models.JSONField(blank=True, null=True) location_details = models.OneToOneField( "LocationDetails", @@ -118,6 +120,11 @@ class Appointment(ModelWithUUID): suffix = str(uuid.uuid4())[:8] self.slug = f"{base_slug}-{suffix}" + if self.geolocation: + self.geolocation_geojson = geojson_from_geolocation(self.geolocation) + else: + self.geolocation_geojson = None + super().save(*args, **kwargs) def delete(self, *args, **kwargs) -> None: @@ -188,9 +195,19 @@ class LocationDetails(ModelWithUUID): ) geolocation = models.PointField(blank=True, null=True, spatial_index=True) + # also store as GeoJSON as downstream processes (Google Datastream) can't understand Postgis binary format + geolocation_geojson = models.JSONField(blank=True, null=True) city = models.CharField(max_length=settings.APPOINTMENT_LOCATION_MAX_LENGTH, blank=True, null=True) state = models.CharField(max_length=settings.APPOINTMENT_LOCATION_MAX_LENGTH, blank=True, null=True) country = models.CharField(max_length=settings.APPOINTMENT_LOCATION_MAX_LENGTH, blank=True, null=True) + + def save(self, *args, **kwargs): + if self.geolocation: + self.geolocation_geojson = geojson_from_geolocation(self.geolocation) + else: + self.geolocation_geojson = None + + super().save(*args, **kwargs) diff --git a/openbook_auth/migrations/0036_rename_userprofile_id_user_openbook_au_id_773856_idx.py b/openbook_auth/migrations/0036_rename_userprofile_id_user_openbook_au_id_773856_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..73b3be9c95e5d5fb06ee657793fb7d3775709274 --- /dev/null +++ b/openbook_auth/migrations/0036_rename_userprofile_id_user_openbook_au_id_773856_idx.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.7 on 2025-03-18 11:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("openbook_auth", "0035_userprofile_interests_v2_userprofile_skills_v2"), + ] + + operations = [ + migrations.RenameIndex( + model_name="userprofile", + new_name="openbook_au_id_773856_idx", + old_fields=("id", "user"), + ), + ] diff --git a/openbook_auth/models.py b/openbook_auth/models.py index 4018b9616f07c0518c51f9f1e01292609e52d817..d124e12b8afc3bcc13c2c9fcfa73095af572d06b 100644 --- a/openbook_auth/models.py +++ b/openbook_auth/models.py @@ -9,7 +9,6 @@ from typing import Iterable, Set, Callable, Type, Optional, List from uuid import UUID import strawberry -from openbook_terms.helpers import topics_to_topic_v2_slugs, skills_to_skill_v2_slugs from django.contrib.auth.models import AbstractUser, UserManager from django.contrib.auth.validators import UnicodeUsernameValidator from django.contrib.contenttypes.fields import GenericRelation @@ -26,11 +25,10 @@ from django.template.loader import render_to_string from django.utils import timezone from django.utils.translation import gettext_lazy as _ from imagekit.models import ProcessedImageField -from pilkit.processors import ResizeToFill, ResizeToFit +from pilkit.processors import ResizeToFill from rest_framework.authtoken.models import Token from unidecode import unidecode -from openbook_common.tracking import track, TrackingEvent from openbook.settings import USERBLOCK from openbook.utils import DEFAULT_EMAIL_LANGUAGE, get_supported_language_for_email from openbook_auth.checkers import * @@ -40,8 +38,7 @@ from openbook_common.enums import VisibilityType from openbook_common.helpers import ExifRotate, generate_blurhash, PrefixedQ from openbook_common.models import Badge, ModelWithUUID from openbook_common.schema.types import Paged -from openbook_common.skills import slug_to_skill_v2 -from openbook_common.topics import slug_to_topic_v2 +from openbook_common.tracking import track, TrackingEvent from openbook_common.types import GeolocationFeature, ReactableType from openbook_common.utils.helpers import delete_file_field from openbook_common.utils.model_loaders import ( @@ -104,6 +101,7 @@ from openbook_hashtags.queries import ( from openbook_notifications.toggles import mute_notifications from openbook_posts.queries import make_get_hashtag_posts_for_user_with_id_query from openbook_posts.query_collections import get_posts_for_user_collection +from openbook_terms.helpers import topics_to_topic_v2_slugs, skills_to_skill_v2_slugs from openbook_terms.models import Topic, SDG, Skill if typing.TYPE_CHECKING: @@ -5264,9 +5262,7 @@ class UserProfile(ModelWithUUID): verbose_name = _("user profile") verbose_name_plural = _("users profiles") - index_together = [ - ("id", "user"), - ] + indexes = [models.Index(fields=["id", "user"])] # Django's Choices don't constrain other values in the db, it needs to be set here: constraints = [ diff --git a/openbook_common/helpers.py b/openbook_common/helpers.py index 2faad837c16bfdcc84c4218452774db0713e4f10..41cc13e957ac12ceea00f9a93dc5421e6d8ed46a 100644 --- a/openbook_common/helpers.py +++ b/openbook_common/helpers.py @@ -6,6 +6,7 @@ from urllib.parse import urlparse import blurhash import requests import strawberry +from django.contrib.gis.db.models import PointField from PIL import ExifTags, Image from PIL.Image import Transpose from django.conf import settings @@ -163,3 +164,13 @@ def to_input_dict(input): converts a strawberry input object to a dict without null values, so it can be validated """ return {k: v for k, v in strawberry.asdict(input).items() if v != strawberry.UNSET} + + +def geojson_from_geolocation(geolocation: PointField) -> object: + return { + "type": "Point", + "coordinates": [ + geolocation.x, # longitude + geolocation.y, # latitude + ], + } diff --git a/openbook_communities/admin.py b/openbook_communities/admin.py index 3a7e8943f636f2d1d50f8f2dd4754a24ef8b240c..8d09625db638dfc9b388e64f1562b87732b33244 100644 --- a/openbook_communities/admin.py +++ b/openbook_communities/admin.py @@ -42,7 +42,7 @@ class CommunityAdmin(admin.ModelAdmin): readonly_fields = ["name"] list_display = ("name", "title", "creator", "created", "get_num_posts", "get_num_members", "is_deleted") search_fields = ("title", "creator__email", "creator__profile__name", "creator__profile__last_name") - exclude = ["banned_users", "starrers"] + exclude = ["banned_users", "starrers", "geolocation_geojson"] def get_queryset(self, request): qs = super().get_queryset(request) @@ -129,6 +129,7 @@ class CommunityTaskAdmin(admin.ModelAdmin): "slots_taken", "slots_available", ) + exclude = ["geolocation_geojson"] @admin.register(OnboardingStep) diff --git a/openbook_communities/migrations/0046_community_geolocation_geojson_and_more.py b/openbook_communities/migrations/0046_community_geolocation_geojson_and_more.py new file mode 100644 index 0000000000000000000000000000000000000000..f696270268482e0d7f89ff5d01d124272ef0b7b3 --- /dev/null +++ b/openbook_communities/migrations/0046_community_geolocation_geojson_and_more.py @@ -0,0 +1,66 @@ +# Generated by Django 5.0.13 on 2025-03-20 04:42 + +from django.db import migrations, models + +from openbook_common.utils.model_loaders import get_community_model, get_task_model + + +def communities_table_name() -> str: + CommunityModel = get_community_model() + return CommunityModel._meta.db_table + + +def tasks_table_name() -> str: + TasksModel = get_task_model() + return TasksModel._meta.db_table + + +def migration_sql(table_name) -> str: + return f""" + UPDATE {table_name} + SET geolocation_geojson = jsonb_build_object( + 'type', 'Point', + 'coordinates', ARRAY[ + ST_X(geolocation::geometry), + ST_Y(geolocation::geometry) + ] + ) + WHERE geolocation IS NOT NULL; + """ + + +def reverse_migration_sql(table_name) -> str: + return f""" + UPDATE {table_name} + SET geolocation_geojson = NULL; + """ + + +def run_migration(apps, schema_editor): + schema_editor.execute(migration_sql(communities_table_name())) + schema_editor.execute(migration_sql(tasks_table_name())) + + +def reverse_migration(apps, schema_editor): + schema_editor.execute(reverse_migration_sql(tasks_table_name())) + schema_editor.execute(reverse_migration_sql(communities_table_name())) + + +class Migration(migrations.Migration): + dependencies = [ + ("openbook_communities", "0045_task_openbook_co_skills__85edfe_gin"), + ] + + operations = [ + migrations.AddField( + model_name="community", + name="geolocation_geojson", + field=models.JSONField(blank=True, null=True), + ), + migrations.AddField( + model_name="task", + name="geolocation_geojson", + field=models.JSONField(blank=True, null=True), + ), + migrations.RunPython(run_migration, reverse_migration), + ] diff --git a/openbook_communities/models/__init__.py b/openbook_communities/models/__init__.py index f3cb162bb76b041624a8e9e17f90cfab31d118e3..d37a157b7c885416a5a3fd1f853af49bcd12b7f8 100644 --- a/openbook_communities/models/__init__.py +++ b/openbook_communities/models/__init__.py @@ -22,7 +22,7 @@ from pilkit.processors import ResizeToFill, ResizeToFit from openbook.settings import COLOR_ATTR_MAX_LENGTH from openbook_auth.models import User from openbook_common.enums import VisibilityType -from openbook_common.helpers import ExifRotate, generate_blurhash, PrefixedQ +from openbook_common.helpers import ExifRotate, generate_blurhash, PrefixedQ, geojson_from_geolocation from openbook_common.models import ModelWithUUID from openbook_common.utils.helpers import delete_file_field, extract_and_capitalize_domain_name from openbook_common.utils.model_loaders import ( @@ -193,6 +193,8 @@ class Community(ModelWithUUID): topics_v2 = ArrayField(models.CharField(max_length=settings.TERM_SLUG_MAX_LENGTH), blank=True, default=list) sdgs = models.ManyToManyField(SDG, related_name="communities", blank=True, db_index=True) geolocation = models.PointField(blank=True, null=True, spatial_index=True) + # also store as GeoJSON as downstream processes (Google Datastream) can't understand Postgis binary format + geolocation_geojson = models.JSONField(blank=True, null=True) contact_description = models.CharField( max_length=settings.COMMUNITY_CONTACT_DESCRIPTION_MAX_LENGTH, blank=True, @@ -809,6 +811,11 @@ class Community(ModelWithUUID): if self.users_adjective: self.users_adjective = self.users_adjective.title() + if self.geolocation: + self.geolocation_geojson = geojson_from_geolocation(self.geolocation) + else: + self.geolocation_geojson = None + return super(Community, self).save(*args, **kwargs) def delete_notifications(self): @@ -1279,6 +1286,8 @@ class Task(ModelWithUUID): db_index=True, ) geolocation = models.PointField(blank=True, null=True, spatial_index=True) + # also store as GeoJSON as downstream processes (Google Datastream) can't understand Postgis binary format + geolocation_geojson = models.JSONField(blank=True, null=True) regularity = models.CharField( max_length=13, choices=Regularity.choices, @@ -1318,6 +1327,13 @@ class Task(ModelWithUUID): class Meta: indexes = [GinIndex(fields=["skills_v2"])] + def save(self, *args, **kwargs): + if self.geolocation: + self.geolocation_geojson = geojson_from_geolocation(self.geolocation) + else: + self.geolocation_geojson = None + super().save(*args, **kwargs) + @property def is_deleted(self) -> bool: return self.deleted_at is not None diff --git a/openbook_communities/schema/queries.py b/openbook_communities/schema/queries.py index 6e9a22a1277e2cff825bcb4b9ff6bb64c16b9f21..8c5a5febf2846e4048bd100903f0d97d5294aea5 100644 --- a/openbook_communities/schema/queries.py +++ b/openbook_communities/schema/queries.py @@ -5,8 +5,7 @@ import strawberry import strawberry_django from asgiref.sync import async_to_sync from django.core.exceptions import PermissionDenied -from django.db.models import Count, Prefetch, F, Q, Case, When, Value, IntegerField -from django.db.models.lookups import IContains +from django.db.models import Count, Prefetch, Q, Case, When, Value, IntegerField from django.utils.translation import gettext_lazy as _ from strawberry.django.context import StrawberryDjangoContext from strawberry.types import Info as StrawberryInfo @@ -15,7 +14,6 @@ from openbook_auth.models import User as UserModel from openbook_auth.utils import verify_authorized_user from openbook_common.schema.types import GeoJSON, Paged from openbook_common.utils.helpers import is_uuid -from openbook_common.validators import string_not_empty from openbook_communities.models import CollaborationTool as CollaborationToolModel from openbook_communities.models import Community as CommunityModel from openbook_communities.models import SpaceUserConnectionType @@ -406,23 +404,3 @@ class Query: offset, limit, ) - - @strawberry_django.field() - def spaces_by_name(self, info, substring: str, offset: int = 0, limit: int = 5) -> Paged[Space]: - string_not_empty(substring) - - title_query = IContains( - F("title"), - substring, - ) - - return Paged.of( - CommunityModel.objects.annotate(member_count=Count("memberships")) - .filter(title_query) - .filter(is_deleted=False) - # Bug HOLI-6780: added created as second order criterion for deterministic order. - # This still has wrong order if while paging the membership counts change. Signed off by Max. - .order_by("-member_count", "-created"), - offset, - limit, - ) diff --git a/openbook_connections/migrations/0003_rename_connection_target_user_target_connection_openbook_co_target__799eff_idx_and_more.py b/openbook_connections/migrations/0003_rename_connection_target_user_target_connection_openbook_co_target__799eff_idx_and_more.py new file mode 100644 index 0000000000000000000000000000000000000000..02dc264c27640de8ad384a66d39a768b5ac7ed85 --- /dev/null +++ b/openbook_connections/migrations/0003_rename_connection_target_user_target_connection_openbook_co_target__799eff_idx_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.7 on 2025-03-18 11:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("openbook_connections", "0002_connection_created"), + ] + + operations = [ + migrations.RenameIndex( + model_name="connection", + new_name="openbook_co_target__799eff_idx", + old_fields=("target_user", "target_connection"), + ), + migrations.RenameIndex( + model_name="connection", + new_name="openbook_co_target__fdd187_idx", + old_fields=("target_user", "id"), + ), + ] diff --git a/openbook_connections/models.py b/openbook_connections/models.py index ed95997a4f494ef0a6a6fc781267d42c15694a99..fa572f2fbdd896bd9abee44ea5d3eefee219d321 100644 --- a/openbook_connections/models.py +++ b/openbook_connections/models.py @@ -13,9 +13,9 @@ class Connection(ModelWithUUID): class Meta: unique_together = ("user", "target_user") - index_together = [ - ("target_user", "target_connection"), - ("target_user", "id"), + indexes = [ + models.Index(fields=["target_user", "target_connection"]), + models.Index(fields=["target_user", "id"]), ] @classmethod diff --git a/openbook_notifications/django_rq_jobs.py b/openbook_notifications/django_rq_jobs.py index 1fe909440b834cdcbb98bc694aa448282b0df8cb..d15001f0d4f67afd5a78683cb1cde1a1ba636c58 100644 --- a/openbook_notifications/django_rq_jobs.py +++ b/openbook_notifications/django_rq_jobs.py @@ -1,10 +1,12 @@ # ruff: noqa from hashlib import sha256 + from django_rq import job from openbook_common.utils.model_loaders import get_user_model + # TODO Think about what to do with OneSignal # onesignal_client = onesignal_sdk.Client( # app_id=settings.ONE_SIGNAL_APP_ID, @@ -16,7 +18,7 @@ from openbook_common.utils.model_loaders import get_user_model @job("default") def send_notification_to_user_with_id(user_id, notification): User = get_user_model() - user = User.objects.only("username", "uuid", "id").get(pk=user_id) + user = User.objects.get(pk=user_id) for device in user.devices.all(): notification.set_parameter("ios_badgeType", "Increase") diff --git a/openbook_posts/migrations/0030_rename_post_creator_community_openbook_po_creator_3cd485_idx.py b/openbook_posts/migrations/0030_rename_post_creator_community_openbook_po_creator_3cd485_idx.py new file mode 100644 index 0000000000000000000000000000000000000000..84a0d8f1267986d23aed90f05a36c190d83a8ecd --- /dev/null +++ b/openbook_posts/migrations/0030_rename_post_creator_community_openbook_po_creator_3cd485_idx.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.7 on 2025-03-18 11:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("openbook_posts", "0029_alter_post_language_code"), + ] + + operations = [ + migrations.RenameIndex( + model_name="post", + new_name="openbook_po_creator_3cd485_idx", + old_fields=("creator", "community"), + ), + ] diff --git a/openbook_posts/migrations/0031_alter_postimage_height_alter_postimage_width.py b/openbook_posts/migrations/0031_alter_postimage_height_alter_postimage_width.py new file mode 100644 index 0000000000000000000000000000000000000000..a6df56b171d1d7ca620763b7df87580e10d34b92 --- /dev/null +++ b/openbook_posts/migrations/0031_alter_postimage_height_alter_postimage_width.py @@ -0,0 +1,23 @@ +# Generated by Django 5.1.5 on 2025-03-20 10:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("openbook_posts", "0030_rename_post_creator_community_openbook_po_creator_3cd485_idx"), + ] + + operations = [ + migrations.AlterField( + model_name="postimage", + name="height", + field=models.PositiveIntegerField(blank=True, editable=False, null=True), + ), + migrations.AlterField( + model_name="postimage", + name="width", + field=models.PositiveIntegerField(blank=True, editable=False, null=True), + ), + ] diff --git a/openbook_posts/models.py b/openbook_posts/models.py index 51119604b89161a7b9d8f0519609ed6a9c56671a..24856e5828791841893c8a7b4be0cb0b87abeda6 100644 --- a/openbook_posts/models.py +++ b/openbook_posts/models.py @@ -39,6 +39,7 @@ from openbook_common.models import ( ModelWithUUID, OrderedModelWithUUID, ) +from openbook_common.tracking import track, TrackingEvent from openbook_common.types import LanguageCode from openbook_common.utils.helpers import ( delete_file_field, @@ -46,7 +47,6 @@ from openbook_common.utils.helpers import ( extract_usernames_from_string, get_magic, ) -from openbook_common.tracking import track, TrackingEvent from openbook_common.utils.model_loaders import ( get_community_model, get_community_new_post_notification_model, @@ -132,9 +132,7 @@ class Post(CommentNotificationMixin, ModelWithUUID): ) class Meta: - index_together = [ - ("creator", "community"), - ] + indexes = [models.Index(fields=["creator", "community"])] @classmethod def post_with_id_has_public_reactions(cls, post_id): @@ -657,8 +655,10 @@ class Post(CommentNotificationMixin, ModelWithUUID): return self def delete(self, *args, **kwargs): - self.delete_media() super(Post, self).delete(*args, **kwargs) + # can only be deleted after deleting the post, + # because Django accesses the underlying file on the super.delete call + self.delete_media() def delete_media(self): if self.has_image(): @@ -835,7 +835,7 @@ class Post(CommentNotificationMixin, ModelWithUUID): username = username.lower() if username not in existing_mention_usernames: try: - user = User.objects.only("id", "username").get(username__iexact=username) + user = User.objects.get(username__iexact=username) user_is_post_creator = user.pk == self.creator_id if not user_is_post_creator: PostUserMention.create_post_user_mention( @@ -1051,8 +1051,8 @@ class PostImage(ModelWithUUID): max_length=settings.IMAGE_URL_MAX_LENGTH, ) image_blurhash = models.CharField(max_length=50, blank=True, null=True) - width = models.PositiveIntegerField(editable=False, null=False, blank=False) - height = models.PositiveIntegerField(editable=False, null=False, blank=False) + width = models.PositiveIntegerField(editable=False, null=True, blank=False) + height = models.PositiveIntegerField(editable=False, null=True, blank=False) media = GenericRelation(PostMedia) @classmethod diff --git a/pytest.ini b/pytest.ini index 04e68df72a680ca027504139e326c1e8f0e47961..51a4d30255362a77892ea9990207878a083964b7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +asyncio_default_fixture_loop_scope = function python_files = test_*.py python_classes = *Tests Test* addopts = @@ -15,4 +16,4 @@ addopts = --ignore=static --ignore=templates --ignore=terraform - --ignore=utils \ No newline at end of file + --ignore=utils diff --git a/renovate.json b/renovate.json index 02856b7994a5a31918c69901b5941b0534b1582d..258a384a2c08d73580c16db2dd3d9450607feb4b 100644 --- a/renovate.json +++ b/renovate.json @@ -23,6 +23,18 @@ ], "groupName": "python" }, + { + "matchJsonata": [ + "$match(depName,/opentelemetry$/)" + ], + "groupName": "opentelemetry" + }, + { + "matchJsonata": [ + "$match(depName,/psycopg$/)" + ], + "groupName": "psycopg" + }, { "matchManagers": [ "pip_requirements" diff --git a/requirements.txt b/requirements.txt index 9472ab6f45718625d9e20e471747b13ddb44dea4..8a485af77f5727e00119afb6cf202b821c23a5d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,13 +7,13 @@ # Afterwards, some packages might need a downgrade or version tweak for the bundle fitting together and fitting the code adrf==0.1.9 aiofiles==24.1.0 -aiohappyeyeballs==2.5.0 -aiohttp==3.11.13 +aiohappyeyeballs==2.6.1 +aiohttp==3.11.14 aiosignal==1.3.2 ASGIMiddlewareStaticFile==0.6.1 asgiref==3.8.1 async-property==0.2.2 -attrs==25.1.0 +attrs==25.3.0 backoff==2.2.1 beautifulsoup4==4.13.3 black==25.1.0 @@ -23,9 +23,9 @@ certifi==2025.1.31 cffi==1.17.1 charset-normalizer==3.4.1 colorama==0.4.6 -coverage==7.6.12 +coverage==7.7.1 Deprecated==1.2.18 -Django==5.0.13 +Django==5.1.7 django-admin-rangefilter==0.13.2 django-appconf==1.1.0 django-cacheops==7.1 @@ -35,7 +35,7 @@ django-debug-toolbar==5.0.1 django-extensions==3.2.3 django-imagekit==5.0.0 django-ipware==7.0.1 -django-modeltranslation==0.19.12 +django-modeltranslation==0.19.13 django-ordered-model==3.7.4 django-proxy==1.3.0 django-redis==5.4.0 @@ -46,49 +46,49 @@ djangorestframework==3.15.2 djangorestframework-camel-case==1.4.2 execnet==2.1.1 Faker==12.0.1 # mixer 7.2.2 depends on Faker<12.1 and >=5.4.0 -filelock==3.17.0 +filelock==3.18.0 frozenlist==1.5.0 funcy==2.0 google-api-core==2.24.2 google-auth==2.38.0 -google-cloud-pubsub==2.28.0 -google-cloud-webrisk==1.17.0 -googleapis-common-protos==1.69.1 +google-cloud-pubsub==2.29.0 +google-cloud-webrisk==1.17.1 +googleapis-common-protos==1.69.2 graphql-core==3.2.6 -grpc-google-iam-v1==0.14.1 +grpc-google-iam-v1==0.14.2 grpcio==1.71.0 grpcio-status==1.71.0 h11==0.14.0 hiredis==3.1.0 -icalendar==6.1.1 +icalendar==6.1.2 idna==3.10 imagekitio==2.2.8 # version 3 contains many breaking changes -importlib_metadata==8.5.0 -iniconfig==2.0.0 +importlib_metadata==8.6.1 +iniconfig==2.1.0 Jinja2==3.1.6 langdetect==1.0.9 log-symbols==0.0.14 MarkupSafe==3.0.2 mixer==7.2.2 monotonic==1.6 -multidict==6.1.0 +multidict==6.2.0 mypy-extensions==1.0.0 novu==1.14.0 -opentelemetry-api==1.30.0 -opentelemetry-sdk==1.30.0 -opentelemetry-semantic-conventions==0.51b0 +opentelemetry-api==1.31.1 +opentelemetry-sdk==1.31.1 +opentelemetry-semantic-conventions==0.52b1 packaging==24.2 pathspec==0.12.1 pilkit==3.0 pillow==11.1.0 -platformdirs==4.3.6 +platformdirs==4.3.7 pluggy==1.5.0 -posthog==3.19.1 -propcache==0.3.0 +posthog==3.21.0 +propcache==0.3.1 proto-plus==1.26.1 -protobuf==5.29.3 -psycopg==3.2.5 -psycopg-binary==3.2.5 +protobuf==5.29.4 +psycopg==3.2.6 +psycopg-binary==3.2.6 pyasn1==0.6.1 pyasn1_modules==0.4.1 pycparser==2.22 @@ -100,35 +100,35 @@ pytest-django==4.10.0 pytest-xdist==3.6.1 python-benedict==0.34.1 python-dateutil==2.9.0.post0 -python-dotenv==1.0.1 +python-dotenv==1.1.0 python-fsutil==0.15.0 python-ipware==3.0.0 python-magic==0.4.27 python-slugify==8.0.4 -pytz==2025.1 +pytz==2025.2 redis==5.2.1 requests==2.32.3 requests-file==2.1.0 requests-toolbelt==1.0.0 rest-framework-generic-relations==2.2.0 -rq==2.1.0 +rq==2.2.0 rsa==4.9 ruamel.yaml==0.18.10 ruamel.yaml.clib==0.2.12 ruff==0.9.10 -sentry-sdk==2.22.0 +sentry-sdk==2.24.1 six==1.17.0 soupsieve==2.6 spinners==0.0.24 sqlparse==0.5.3 strawberry-graphql==0.248.1 strawberry-graphql-django==0.53.3 -structlog==25.1.0 +structlog==25.2.0 termcolor==2.5.0 text-unidecode==1.3 tldextract==5.1.3 -typing_extensions==4.12.2 -tzdata==2025.1 +typing_extensions==4.13.0 +tzdata==2025.2 Unidecode==1.3.8 uritools==4.0.3 url-normalize==1.4.3 diff --git a/terraform/common/init.tf b/terraform/common/init.tf index 1c76b767fcd176e75b421ac88b0aaf5f003f4064..02eb405058af2dde807bd6d21e5c92d61b0e6db8 100644 --- a/terraform/common/init.tf +++ b/terraform/common/init.tf @@ -4,11 +4,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "6.24.0" + version = "6.26.0" } google-beta = { source = "hashicorp/google-beta" - version = "6.24.0" + version = "6.26.0" } } backend "gcs" { diff --git a/terraform/environments/init.tf b/terraform/environments/init.tf index 85e6590c4d2906610cf0fa111d4a73ae5b62962f..6a5dcb1c23fc5ac56708360336c34664308df26e 100644 --- a/terraform/environments/init.tf +++ b/terraform/environments/init.tf @@ -4,11 +4,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = "6.24.0" + version = "6.26.0" } google-beta = { source = "hashicorp/google-beta" - version = "6.24.0" + version = "6.26.0" } } backend "gcs" {