diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 47f48832d0f32bbe5069b5fa15d01d6cb12ffe6a..bc71ac812333251dbfd326a8e9b9f983c5511ecc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,24 +1,21 @@ variables: - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # Change pip's cache directory to be inside the project directory since we can only cache local items. - API_DOMAIN_PATH: "$CI_PROJECT_DIR/api_domain" + PIP_CACHE_DIR: '$CI_PROJECT_DIR/.cache/pip' # Change pip's cache directory to be inside the project directory since we can only cache local items. + API_DOMAIN_PATH: '$CI_PROJECT_DIR/api_domain' cache: - key: - files: - - requirements.txt # hashes this file so cache content depends on files content + key: $CI_COMMIT_REF_SLUG # using requirements.txt hash led to too many cache files paths: - - /root/.cache/pip - .cache/pip - venv/ stages: - - "setup" - - "test" - - "build" - - "deploy" - - "smoketest" - - "downstream" - - "destroy" + - 'setup' + - 'test' + - 'build' + - 'deploy' + - 'smoketest' + - 'downstream' + - 'destroy' default: before_script: @@ -31,8 +28,8 @@ default: - 1cpu-4gb # build on smaller machine setup: - image: "europe-north1-docker.pkg.dev/holi-shared/docker/holi-docker/holi-okuna-builder" - stage: "setup" + image: 'europe-north1-docker.pkg.dev/holi-shared/docker/holi-docker/holi-okuna-builder' + stage: 'setup' script: - python3 -m venv venv - source venv/bin/activate @@ -40,13 +37,13 @@ setup: - pip install -r requirements.txt --exists-action s unit-test: - image: "europe-north1-docker.pkg.dev/holi-shared/docker/holi-docker/holi-okuna-builder" + image: 'europe-north1-docker.pkg.dev/holi-shared/docker/holi-docker/holi-okuna-builder' services: - - name: 'europe-north1-docker.pkg.dev/holi-shared/docker-hub-remote/postgis/postgis:12-3.3-alpine' + - name: 'europe-north1-docker.pkg.dev/holi-shared/docker-hub-remote/postgis/postgis:17-3.4' alias: postgres - stage: "test" + stage: 'test' variables: - SECRET_KEY: "secret" + SECRET_KEY: 'secret' POSTGRES_DB: okuna_test_db POSTGRES_USER: okuna_test_user POSTGRES_PASSWORD: okuna_test_pass @@ -59,11 +56,11 @@ unit-test: - 4cpu-4gb # more cpu, quicker testing lint: - image: "europe-north1-docker.pkg.dev/holi-shared/docker/holi-docker/holi-okuna-builder" + image: 'europe-north1-docker.pkg.dev/holi-shared/docker/holi-docker/holi-okuna-builder' stage: test variables: GIT_STRATEGY: clone - GIT_CHECKOUT: "false" + GIT_CHECKOUT: 'false' before_script: - git fetch --all - git checkout $CI_COMMIT_REF_NAME @@ -83,9 +80,10 @@ lint: build_docker: image: 'europe-north1-docker.pkg.dev/holi-shared/docker-hub-remote/docker:27' - stage: "build" + stage: 'build' services: - - 'docker:27-dind' + - name: 'europe-north1-docker.pkg.dev/holi-shared/docker-hub-remote/docker:27-dind' + alias: 'docker' variables: # this could be fetched via terraform output ("gcr_location" in infra project), but then we would need an extra job for terraform GCR_IMAGE: europe-north1-docker.pkg.dev/holi-shared/docker/holi-okuna @@ -99,21 +97,22 @@ build_docker: - 4cpu-4gb # more cpu, quicker image build .deploy: - stage: "deploy" + stage: 'deploy' image: - name: "europe-north1-docker.pkg.dev/holi-shared/docker/holi-docker/terraform-gcloud" + name: 'europe-north1-docker.pkg.dev/holi-shared/docker/holi-docker/terraform-gcloud' # default entrypoint is terraform command, but we want to run shell scripts # entrypoint: ["/bin/sh", "-c"] variables: ENVIRONMENT_ID: $CI_ENVIRONMENT_SLUG artifacts: paths: - - "terraform/environments/crash.log" # optional, only available in case of a crash/panic - - "terraform/environments/terraform-*.log" # separate log for every step/command + - 'terraform/environments/crash.log' # optional, only available in case of a crash/panic + - 'terraform/environments/terraform-*.log' # separate log for every step/command - $API_DOMAIN_PATH - name: "${CI_JOB_NAME}_${CI_JOB_ID}" + name: '${CI_JOB_NAME}_${CI_JOB_ID}' #when: on_failure # can't do that for api base url, but can't define multiple artifacts expire_in: 1 week + cache: [] # explicitly disable cache script: # gcloud auth is needed for the migration, triggered via gcloud command - gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS @@ -123,8 +122,9 @@ build_docker: interruptible: false .smoketest: - stage: "smoketest" + stage: 'smoketest' image: 'europe-north1-docker.pkg.dev/holi-shared/docker/holi-docker/holi-k6-builder' + cache: [] # explicitly disable cache script: - API_DOMAIN=$(cat $API_DOMAIN_PATH) - terraform/environments/scripts/wait-for-ssl.sh "https://${API_DOMAIN}" @@ -153,9 +153,9 @@ review_smoketest: review_destroy: stage: destroy image: - name: 'europe-north1-docker.pkg.dev/holi-shared/docker-hub-remote/hashicorp/terraform:1.6.6' + name: 'europe-north1-docker.pkg.dev/holi-shared/docker-hub-remote/hashicorp/terraform:1.9.8' # default entrypoint is terraform command, but we want to run shell scripts - entrypoint: ["/bin/sh", "-c"] + entrypoint: ['/bin/sh', '-c'] variables: # has to be set to none for auto stop GIT_STRATEGY: none @@ -163,11 +163,12 @@ review_destroy: name: review/$CI_COMMIT_REF_SLUG action: stop dependencies: [] # explicitly disable artifact usage + cache: [] # explicitly disable cache artifacts: paths: - - "terraform/environments/crash.log" # optional, only available in case of a crash/panic - - "terraform/environments/terraform-*.log" # separate log for every step/command - name: "${CI_JOB_NAME}_${CI_JOB_ID}" + - 'terraform/environments/crash.log' # optional, only available in case of a crash/panic + - 'terraform/environments/terraform-*.log' # separate log for every step/command + name: '${CI_JOB_NAME}_${CI_JOB_ID}' when: on_failure expire_in: 1 week script: diff --git a/.terraform-version b/.terraform-version index 9c6d6293b1a8f448def89c2d5bfa63b89a24e0cc..66beabb5795e7441046623a209a2c48f22e7d20a 100644 --- a/.terraform-version +++ b/.terraform-version @@ -1 +1 @@ -1.6.1 +1.9.8 diff --git a/docker-compose-full.yml b/docker-compose-full.yml index 8efb22fbad4feb5e35d6f7f6e4f6e47df3c1221d..14732b5dc7a0fa29d53f7fb1ef14b155839b22fb 100644 --- a/docker-compose-full.yml +++ b/docker-compose-full.yml @@ -67,7 +67,7 @@ services: env_file: - .docker-compose.env db: - image: postgis/postgis:12-3.3-alpine + image: postgis/postgis:17-3.4 container_name: okuna-postgres hostname: db.okuna volumes: diff --git a/docker-compose-services-only.yml b/docker-compose-services-only.yml index 8feed3350628201467b5b0d52cf015351b64f412..7427ae5238c2f75f7ab511199393422fd468c161 100644 --- a/docker-compose-services-only.yml +++ b/docker-compose-services-only.yml @@ -2,7 +2,7 @@ version: '3' services: db: - image: ${POSTGRES_IMAGE:-postgis/postgis:12-3.3-alpine} + image: ${POSTGRES_IMAGE:-europe-north1-docker.pkg.dev/holi-shared/docker-hub-remote/postgis/postgis:17-3.4} container_name: okuna-postgres hostname: db.okuna volumes: @@ -16,7 +16,7 @@ services: env_file: - .docker-compose.env redis: - image: redis:alpine + image: redis command: redis-server /usr/local/etc/redis/redis.conf --requirepass ${REDIS_PASSWORD} --port 6379 container_name: okuna-redis privileged: false diff --git a/renovate.json b/renovate.json index 4c06317c105cb37ea99349699b427d034a2a4bcb..c9dcc364f880e05aa4b36d4fd9fb0f71f7f3b739 100644 --- a/renovate.json +++ b/renovate.json @@ -1,17 +1,16 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ - "config:recommended" + "config:recommended", + ":configMigration", + ":label('renovatebot')", + ":enableVulnerabilityAlertsWithLabel('renovatebot/security')", + ":semanticCommits", + ":automergePatch", + ":automergeTesters", + ":automergeLinters" ], "packageRules": [ - { - "matchUpdateTypes": [ - "patch", - "pin", - "digest" - ], - "automerge": true - }, { "matchSourceUrls": [ "https://github.com/hashicorp/terraform" @@ -34,7 +33,7 @@ "pin", "digest" ], - "automerge": true + "automerge": false }, { "matchDepNames": [ @@ -43,12 +42,25 @@ ], "groupName": "boto3" }, + { + "matchDepNames": [ + "pydantic", + "pydantic_core" + ], + "groupName": "pydantic" + }, + { + "matchDepNames": [ + "mixer", + "faker" + ], + "groupName": "mixer&faker" + }, { "matchDepTypes": [ "devDependencies" ], "automerge": true } - ], - "platformAutomerge": false + ] } \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 43ac9469cae1878acdeccb0c6e1111819ae9492b..97520281059af1a45daf0902a81572dc20f49eba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,41 +1,35 @@ # For upgrading all dependencies to their highest compatible versions: # -# sed -i '' 's/[~=]=/>=/' requirements.txt # replace all version specifiers with a companion specifier +# sed -i 's/[~=]=/>=/' requirements.txt # replace all version specifiers with a companion specifier # pip install -U -r requirements.txt # now upgrade all packages in requirements.txt # pip freeze | sed 's/==/~=/' > requirements.txt # freeze adds any transient dependencies and makes them explicit # # then, we transform the exact matches to compatibility matches # # Afterwards, some packages might need a downgrade or version tweak for the bundle fitting together and fitting the code -adrf~=0.1.7 +adrf~=0.1.8 aiofiles~=24.1.0 -aiohttp~=3.10.1 +aiohappyeyeballs~=2.4.3 +aiohttp~=3.10.10 aiosignal~=1.3.1 -annotated-types~=0.7.0 -anyio~=4.6.2.post1 -appdirs~=1.4.4 ASGIMiddlewareStaticFile~=0.6.1 asgiref~=3.8.1 async-property~=0.2.2 attrs~=24.2.0 -Authlib~=1.3.1 -bandit~=1.7.9 +backoff~=2.2.1 beautifulsoup4~=4.12.3 black~=24.10.0 blurhash-python~=1.2.2 -boto3~=1.35.46 -botocore~=1.35.46 cachetools~=5.5.0 certifi~=2024.8.30 -cffi~=1.17.0 +cffi~=1.17.1 charset-normalizer~=3.4.0 -click~=8.1.3 +click~=8.1.7 colorama~=0.4.6 colorlog~=6.8.2 -coverage~=7.6.1 -cryptography~=43.0.0 -decorator~=5.1.1 +coverage~=7.6.4 +Deprecated~=1.2.14 Django~=5.0.7 -django-admin-rangefilter~=0.13.1 +django-admin-rangefilter~=0.13.2 django-appconf~=1.0.6 django-cacheops~=7.0.2 django-cors-headers~=4.5.0 @@ -44,9 +38,7 @@ django-debug-toolbar~=4.4.6 django-extensions~=3.2.3 django-imagekit~=5.0.0 django-ipware~=7.0.1 -django-model-utils~=4.5.1 -django-modeltranslation~=0.19.5 -django-nose~=1.4.7 +django-modeltranslation~=0.19.10 django-ordered-model~=3.7.4 django-proxy~=1.3.0 django-redis~=5.4.0 @@ -55,121 +47,98 @@ django-sortedm2m~=4.0.0 django-structlog~=8.1.0 djangorestframework~=3.15.2 djangorestframework-camel-case~=1.4.2 -dparse~=0.6.4b0 execnet~=2.1.1 Faker~=12.0.1 # mixer 7.2.2 depends on Faker<12.1 and >=5.4.0 filelock~=3.16.1 frozenlist~=1.5.0 funcy~=2.0 -gitdb~=4.0.11 -GitPython~=3.1.43 google-api-core~=2.21.0 google-auth~=2.35.0 google-cloud-pubsub~=2.26.1 -google-cloud-webrisk~=1.14.5 +google-cloud-webrisk~=1.15.0 googleapis-common-protos~=1.65.0 -graphql-core~=3.2.3 -grpc-google-iam-v1~=0.13.0 +graphql-core~=3.2.5 +grpc-google-iam-v1~=0.13.1 grpcio~=1.67.0 -grpcio-status~=1.62.1 +grpcio-status~=1.62.3 h11~=0.14.0 halo~=0.0.31 hiredis~=3.0.0 -httpcore~=1.0.5 -httpx~=0.27.0 -icalendar~=5.0.13 -idna~=3.6 +icalendar~=6.0.1 +idna~=3.10 imagekitio==2.2.8 # version 3 contains many breaking changes +importlib_metadata~=8.4.0 iniconfig~=2.0.0 Jinja2~=3.1.4 -jmespath~=1.0.1 langdetect~=1.0.9 log-symbols~=0.0.14 -markdown-it-py~=3.0.0 -MarkupSafe~=2.1.5 -marshmallow~=3.23.0 -mdurl~=0.1.2 +MarkupSafe~=3.0.2 mixer~=7.2.2 +monotonic~=1.6 multidict~=6.1.0 mypy-extensions~=1.0.0 -nose~=1.3.7 -nose-exclude~=0.5.0 novu~=1.14.0 +opentelemetry-api~=1.27.0 +opentelemetry-sdk~=1.27.0 +opentelemetry-semantic-conventions~=0.48b0 packaging~=24.1 pathspec~=0.12.1 -pathtools~=0.1.2 -pbr~=6.1.0 pilkit~=3.0 -pillow~=10.4.0 -pinocchio~=0.4.3 +pillow~=11.0.0 platformdirs~=4.3.6 pluggy~=1.5.0 -posthog==3.7.0 +posthog~=3.7.0 +propcache~=0.2.0 proto-plus~=1.25.0 -protobuf~=4.25.3 -psycopg~=3.2.1 -psycopg-binary~=3.2.1 -pyasn1~=0.6.0 -pyasn1_modules~=0.4.0 -pycparser~=2.21 -pydantic~=2.8.2 -pydantic_core~=2.20.1 -Pygments~=2.18.0 +protobuf~=5.28.3 +psycopg~=3.2.3 +psycopg-binary~=3.2.3 +pyasn1~=0.6.1 +pyasn1_modules~=0.4.1 +pycparser~=2.22 PyJWT~=2.9.0 -pyparsing~=3.2.0 -pytest~=8.3.2 +pytest~=8.3.3 pytest-asyncio~=0.24.0 pytest-cov~=5.0.0 pytest-django~=4.9.0 pytest-xdist~=3.6.1 -python-benedict~=0.33.2 +python-benedict~=0.34.0 python-dateutil~=2.9.0.post0 python-dotenv~=1.0.1 python-fsutil~=0.14.1 python-ipware~=3.0.0 python-magic~=0.4.27 python-slugify~=8.0.4 -pytz~=2024.1 -PyYAML~=6.0.1 -redis~=5.0.8 +pytz~=2024.2 +redis~=5.2.0 requests~=2.32.3 requests-file~=2.1.0 requests-toolbelt~=0.10.1 rest-framework-generic-relations~=2.2.0 -rich~=13.7.1 rq~=1.16.2 rsa~=4.9 ruamel.yaml~=0.18.6 -ruamel.yaml.clib~=0.2.8 -ruff~=0.5.6 -s3transfer~=0.10.2 -safety~=3.2.3 -safety-schemas~=0.0.2 -sentry-sdk~=2.12.0 -shellingham~=1.5.4 -shutilwhich~=1.1.0 +ruamel.yaml.clib~=0.2.12 +ruff~=0.7.1 +sentry-sdk~=2.17.0 six~=1.16.0 -smmap~=5.0.1 -sniffio~=1.3.1 -soupsieve~=2.5 +soupsieve~=2.6 spinners~=0.0.24 sqlparse~=0.5.1 -stevedore~=5.2.0 -strawberry-graphql~=0.237.3 -strawberry-graphql-django~=0.47.1 +strawberry-graphql~=0.247.0 +strawberry-graphql-django~=0.49.1 structlog~=24.4.0 -termcolor~=2.4.0 +termcolor~=2.5.0 text-unidecode~=1.3 tldextract~=5.1.2 -toml~=0.10.2 -typer~=0.12.3 typing_extensions~=4.12.2 +tzdata~=2024.2 Unidecode~=1.3.8 uritools~=4.0.3 url-normalize~=1.4.3 urlextract~=1.9.0 -urllib3~=1.26.19 # imagekit requires <1.27 -uvicorn~=0.30.5 -watchdog~=4.0.1 -yarl~=1.9.4 +urllib3~=1.26.20 +uvicorn~=0.32.0 +wrapt~=1.16.0 +yarl~=1.16.0 zipp~=3.20.2 diff --git a/terraform/environments/scripts/create-or-update-env.sh b/terraform/environments/scripts/create-or-update-env.sh index 89d2951406f8c790079204b1a0fe0d1308130939..0a0e9df84968a168a5b6416f31f29335fa8c333c 100755 --- a/terraform/environments/scripts/create-or-update-env.sh +++ b/terraform/environments/scripts/create-or-update-env.sh @@ -13,6 +13,6 @@ TAG=$2 TF_LOG_PATH=terraform-init.log terraform init TF_LOG_PATH=terraform-version.log terraform version -TF_LOG_PATH=terraform-workspace.log terraform workspace new "$WORKSPACE" || terraform workspace select "$WORKSPACE" +TF_LOG_PATH=terraform-workspace.log terraform workspace select -or-create=true "$WORKSPACE" TF_LOG_PATH=terraform-plan.log terraform plan -var="image_tag=$TAG" -out plan -no-color | tee tfplan.plain TF_LOG_PATH=terraform-apply.log terraform apply -auto-approve -parallelism=50 plan \ No newline at end of file