diff --git a/.envrc.local.template b/.envrc.local.template
index f0c3c0d61ebb2ab380de3384d49342189ebd62af..b175c0221ee15e15641b3b36b31cb1f161a60e50 100644
--- a/.envrc.local.template
+++ b/.envrc.local.template
@@ -10,3 +10,5 @@ export DB_USERNAME=
export DB_PASSWORD=
export GEO_API_ENDPOINT_URL=
+# Using local backends:
+# export GEO_API_ENDPOINT_URL=http://localhost:8003/graphql
diff --git a/README.md b/README.md
index 4a1a188f3d1dafcf06609e5c69d35b8de5ba5715..dffc127ed2643fd84bc3d3d17131a41cdb87d9df 100644
--- a/README.md
+++ b/README.md
@@ -194,12 +194,13 @@ if "you know what you're doing".
### Configuration
-| Environment Variable | Default Value | Description |
-|----------------------------------| ---------------------------------------------------------------------------------------- |----------------------------------------|
-| PORT | 8004 | the port to listen on |
-| CACHE_ENABLED | true | wether or not to enable caching |
-| CACHE_TTL_MS | 60 seconds | time-to-live in ms |
-| VOLUNTEERING_VOLTASTICS_API_URL | undefined | Voltastics API base URL |
-| VOLUNTEERING_VOLTASTICS_API_KEY | undefined | Voltastics API Token |
-| IMAGE_PROXY_BASE_URL | https://images.holi.social (production), https://dev-images.holi.social (all other envs) | Base URL for the image proxy server |
-| GEO_API_ENDPOINT_URL | undefined | GraphQL Endpoint URL for holis Geo API |
+| Environment Variable | Default Value | Description |
+| ------------------------------- | -------------------------------------------------------------------------------------------- | ------------------------------------------------------- |
+| PORT | 8004 | the port to listen on |
+| CACHE_ENABLED | true | whether or not to enable caching |
+| CACHE_TTL_MS_VOLTASTICS | 24 hours | time-to-live in ms for data fetched from Voltastics API |
+| CACHE_TTL_MS_DB | 1 hour | time-to-live in ms for data fetched from DB |
+| VOLUNTEERING_VOLTASTICS_API_URL | undefined | Voltastics API base URL |
+| VOLUNTEERING_VOLTASTICS_API_KEY | undefined | Voltastics API Token |
+| IMAGE_PROXY_BASE_URL | <https://images.holi.social> (production), <https://dev-images.holi.social> (all other envs) | Base URL for the image proxy server |
+| GEO_API_ENDPOINT_URL | undefined | GraphQL Endpoint URL for holis Geo API |
diff --git a/app/api_types.ts b/app/api_types.ts
index 315a01245d3f21a83d4596a356950733b887febc..e62283333d10d8ed611b30aad335c94b981284e3 100644
--- a/app/api_types.ts
+++ b/app/api_types.ts
@@ -31,7 +31,6 @@ export enum ApiRoutes {
SEARCH_ENGAGEMENTS = "searchengagement",
TRACK_VIEW = "trackview",
ENGAGEMENT = "engagement",
- CITIES = "cities",
USES = "uses",
}
@@ -40,6 +39,3 @@ export enum ApiDefaults {
MIN_RADIUS = 5,
MAX_RADIUS = 50, // The largest radius in kilometers that the Voltastics API is capable of handling.
}
-
-// TODO: Delete the following when app version 1.5.1 is no longer supported.
-export type ApiCitiesResponse = string[];
diff --git a/app/deps.ts b/app/deps.ts
index 515034b748ff2bf6378e364f24302b1b4f36bf07..5e02d4d53e134f19b2892816e1d1ba184f01726a 100644
--- a/app/deps.ts
+++ b/app/deps.ts
@@ -3,7 +3,4 @@ export { createSchema, createYoga } from "npm:graphql-yoga@5.1.1";
export { useResponseCache } from "npm:@graphql-yoga/plugin-response-cache@1.0.0";
export { GraphQLError } from "npm:graphql@16.8.1";
export * as turf from "https://esm.sh/@turf/turf@6.5.0";
-export {
- Client,
- type QueryObjectResult,
-} from "https://deno.land/x/postgres/mod.ts";
+export { Client, type QueryObjectResult } from "https://deno.land/x/postgres/mod.ts";
diff --git a/app/dev_deps.ts b/app/dev_deps.ts
index dd30673135247255a2a88a65530e2d7b506744f8..59b65bea4e37c96d763fb21465fe0dc579d214e6 100644
--- a/app/dev_deps.ts
+++ b/app/dev_deps.ts
@@ -7,13 +7,5 @@ export {
stub,
} from "https://deno.land/std@0.155.0/testing/mock.ts";
export type { Stub } from "https://deno.land/std@0.155.0/testing/mock.ts";
-export {
- assertEquals,
- assertExists,
- assertRejects,
-} from "https://deno.land/std@0.155.0/testing/asserts.ts";
-export {
- afterEach,
- describe,
- it,
-} from "https://deno.land/std@0.155.0/testing/bdd.ts";
+export { assertEquals, assertExists, assertRejects } from "https://deno.land/std@0.155.0/testing/asserts.ts";
+export { afterEach, describe, it } from "https://deno.land/std@0.155.0/testing/bdd.ts";
diff --git a/app/geo_api_client.ts b/app/geo_api_client.ts
index 427e954c5527a8b44c73cf8d303d8eb5779f5509..6a14360331bbe471a0ccc2efced020d1e36a1a92 100644
--- a/app/geo_api_client.ts
+++ b/app/geo_api_client.ts
@@ -1,24 +1,28 @@
-import { GeoLocation } from "./volunteering_db.ts";
+import { GeolocationCoordinates, GeolocationGeometry } from "./types.ts";
type GeoAPIResponse = {
data?: {
placeDetails?: {
geolocation?: {
- properties?: {
- lat?: number;
- lon?: number;
- };
+ properties?: Partial<GeolocationCoordinates>;
+ geometry?: Partial<GeolocationGeometry>;
};
};
};
};
-export const resolveGeoLocationViaGeoAPI =
- (geoAPIEndpointUrl: string) =>
- async (geolocationId: string): Promise<GeoLocation> => {
- const graphQLQuery =
- `{ placeDetails(id: "${geolocationId}") { geolocation } }`;
- const response = await fetch(geoAPIEndpointUrl, {
+export class GeoAPIClient {
+ private readonly geoAPIEndpointUrl: string;
+
+ constructor(geoAPIEndpointUrl: string) {
+ this.geoAPIEndpointUrl = geoAPIEndpointUrl;
+ }
+
+ private async resolveGeolocationViaGeoAPI(
+ geolocationId: string,
+ ): Promise<GeoAPIResponse> {
+ const graphQLQuery = `{ placeDetails(id: "${geolocationId}") { geolocation } }`;
+ const response = await fetch(this.geoAPIEndpointUrl, {
"body": JSON.stringify({ query: graphQLQuery }),
"headers": {
"Accept": "application/graphql-response+json, application/json",
@@ -26,14 +30,36 @@ export const resolveGeoLocationViaGeoAPI =
},
"method": "POST",
});
- const responseJSON = await response.json() as GeoAPIResponse;
- const { lat, lon } =
- responseJSON.data?.placeDetails?.geolocation?.properties || {};
+ return response.json() as GeoAPIResponse;
+ }
+
+ async resolveCoordinates(
+ geolocationId: string,
+ ): Promise<GeolocationCoordinates> {
+ const responseJSON = await this.resolveGeolocationViaGeoAPI(geolocationId);
+
+ const { lat, lon } = responseJSON.data?.placeDetails?.geolocation?.properties || {};
if (lat && lon) {
return { lat, lon };
} else {
return Promise.reject(
- `Resolution of lat/lng failed (no data included in response for geolocationId=${geolocationId})`,
+ `Resolution of lat/lon failed (no data included in response for geolocationId=${geolocationId})`,
);
}
- };
+ }
+
+ async resolveGeometry(
+ geolocationId: string,
+ ): Promise<GeolocationGeometry> {
+ const responseJSON = await this.resolveGeolocationViaGeoAPI(geolocationId);
+ const { type, coordinates } = responseJSON.data?.placeDetails?.geolocation
+ ?.geometry || {};
+ if (type && coordinates) {
+ return { type, coordinates };
+ } else {
+ return Promise.reject(
+ `Resolution of geolocation geometry failed (no data included in response for geolocationId=${geolocationId})`,
+ );
+ }
+ }
+}
diff --git a/app/helpers.ts b/app/helpers.ts
deleted file mode 100644
index 1d5e1a825346d48f852751ada0a6fe570fc00fe3..0000000000000000000000000000000000000000
--- a/app/helpers.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-export const sortCitiesAlphabetically = (a: string, b: string) =>
- Number(/^\d/.test(a)) - Number(/^\d/.test(b)) ||
- Number(/^"/.test(a)) - Number(/^"/.test(b)) ||
- a.localeCompare(b, "de-DE", { numeric: true, sensitivity: "base" });
diff --git a/app/logging.ts b/app/logging.ts
index 06779e533e520e78df05e7bb026dba44f13d747e..9a7c98004e9d7ed0b099bf873229aaeffb4ac9a3 100644
--- a/app/logging.ts
+++ b/app/logging.ts
@@ -49,14 +49,12 @@ class Logger {
if (!shouldLogLevel(this.level, severity)) return;
const date = new Date().toISOString();
- const input = this.environment === "development"
- ? `${date} ${severity} ${message}`
- : JSON.stringify({
- environment: this.environment,
- severity,
- message,
- ...options,
- });
+ const input = this.environment === "development" ? `${date} ${severity} ${message}` : JSON.stringify({
+ environment: this.environment,
+ severity,
+ message,
+ ...options,
+ });
switch (severity) {
case LogSeverity.DEFAULT:
diff --git a/app/main.ts b/app/main.ts
index c3efe304aa0d37d86fb25b3213c54d0cb87f02fc..de61bffe420f66ac83b35521a9e096562bd80881 100644
--- a/app/main.ts
+++ b/app/main.ts
@@ -1,6 +1,7 @@
import { logger, LogSeverity } from "./logging.ts";
import {
DEFAULT_CACHE_ENABLED,
+ DEFAULT_CACHE_TTL_MS_DB,
DEFAULT_CACHE_TTL_MS_VOLTASTICS,
DEFAULT_DB_CONNECTION_ATTEMPTS,
DEFAULT_PORT,
@@ -44,6 +45,11 @@ const serverConfigFromEnv = (): ServerConfig => {
Number,
DEFAULT_CACHE_TTL_MS_VOLTASTICS,
),
+ cacheTtlMsDB: requiredEnv(
+ "DEFAULT_CACHE_TTL_MS_DB",
+ Number,
+ DEFAULT_CACHE_TTL_MS_DB,
+ ),
voltastics: {
baseUrl: requiredEnv(
"VOLUNTEERING_VOLTASTICS_API_URL",
@@ -59,9 +65,7 @@ const serverConfigFromEnv = (): ServerConfig => {
imageProxyBaseUrl: requiredEnv(
"IMAGE_PROXY_BASE_URL",
String,
- environment === "production"
- ? "https://images.holi.social"
- : "https://dev-images.holi.social",
+ environment === "production" ? "https://images.holi.social" : "https://dev-images.holi.social",
),
volunteeringDB: {
hostname: requiredEnv(
diff --git a/app/server.ts b/app/server.ts
index adcc31d9579eb457d20e920ffbb83be5b010ddc0..380056d166523088f5d0d559927618d7b6a55468 100644
--- a/app/server.ts
+++ b/app/server.ts
@@ -1,27 +1,19 @@
import {
CategoriesResponse,
- CitiesResponse,
EngagementOpportunitiesParameters,
EngagementParameters,
EngagementRecommendationsParameters,
EngagementResponse,
- EngagementsParameters,
EngagementsResponse,
+ FilteredEngagementsParameters,
TrackEngagementViewParameters,
TrackEngagementViewResponse,
} from "./types.ts";
import { createSchema, createYoga, serve, useResponseCache } from "./deps.ts";
-import {
- fetchCategories,
- fetchCities,
- fetchEngagement,
- fetchEngagementOpportunities,
- fetchEngagements,
- trackEngagementView,
-} from "./voltastics.ts";
+import { fetchCategories, fetchEngagement, fetchEngagementOpportunities, trackEngagementView } from "./voltastics.ts";
import { logger } from "./logging.ts";
import { VolunteeringDB, VolunteeringDBConfig } from "./volunteering_db.ts";
-import { resolveGeoLocationViaGeoAPI } from "./geo_api_client.ts";
+import { GeoAPIClient } from "./geo_api_client.ts";
import { Client } from "https://deno.land/x/postgres@v0.19.3/client.ts";
const typeDefs = `
@@ -47,8 +39,22 @@ const typeDefs = `
location: String
latitude: Float
longitude: Float
+
+ """
+ If this engagement is part of a recommendation query response, matchedTopics contains the slugs of matched topics. The order is defined as follows: 1) topics that were part of the query come first (sorted by matching confidence). 2) other topics the engagement matches (sorted by matching confidence)
+ """
+ matchedTopics: [String]
+
+ """
+ If this engagement is part of a recommendation query response, matchedSkills contains the slugs of matched skills. The order is defined as follows: 1) skills that were part of the query come first (sorted by matching confidence). 2) other topics the engagement matches (sorted by matching confidence)
+ """
+ matchedSkills: [String]
+
+ """
+ If this engagement is part of a recommendation query response, matchedGeoLocationDistanceInMeters contains the distance (in meters) between the engagement and the location given in the query.
+ """
+ matchedGeoLocationDistanceInMeters: Float
}
-
type Category {
name: String!
@@ -65,23 +71,19 @@ const typeDefs = `
scalar GeoJSON
- type City {
- name: String!
- }
-
- type CitiesResponse {
- data: [City]!
- }
-
-
type Query {
# uses offset-based pagination as described in https://www.apollographql.com/docs/react/pagination/offset-based
engagementOpportunities(offset: Int! = 0, limit: Int! = 10, location: GeoJSON, category: String): EngagementsResponse!
- engagementRecommendations(offset: Int! = 0, limit: Int! = 10, topics: [String!], skills: [String!], geolocationId: String): EngagementsResponse!
+
+ """
+ Retrieves a list of recommendations based on the request parameters "topics", "skills" and "geoLocationId". "topics" contains slugs of the "Topic" type, "skills" contains slugs of the "Skill" type, "geoLocationId" contains the ID of a location as returned by the "geo_*" queries.
+ """
+ engagementRecommendations(offset: Int! = 0, limit: Int! = 10, topics: [String!] = [], skills: [String!] = [], geolocationId: String): EngagementsResponse!
+
engagement(id: String!): Engagement
+
categories: CategoriesResponse!
- engagements(offset: Int! = 0, limit: Int! = 10, location: String, category: String, longitude: Float, latitude: Float, radiusKm: Float): EngagementsResponse! @deprecated(reason: "Use engagementOpportunities instead, as its support will be discontinued after 1.5.1.")
- cities: CitiesResponse! @deprecated(reason: "It will no longer be supported after 1.5.1.")
+ filteredEngagements(offset: Int! = 0, limit: Int! = 10, topics: [String!] = [], skills: [String!] = [], geolocationId: String): EngagementsResponse!
}
type Mutation {
@@ -99,45 +101,28 @@ const createResolvers = (
_parent: any,
parameters: EngagementOpportunitiesParameters,
): Promise<EngagementsResponse> =>
- config.fake
- ? Promise.resolve({ totalResults: 0, data: [] })
- : fetchEngagementOpportunities(config)(parameters),
+ config.fake ? Promise.resolve({ totalResults: 0, data: [] }) : fetchEngagementOpportunities(config)(parameters),
engagement: (
// deno-lint-ignore no-explicit-any
_parent: any,
parameters: EngagementParameters,
- ): Promise<EngagementResponse> =>
- config.fake ? Promise.resolve(null) : fetchEngagement(config)(parameters),
+ ): Promise<EngagementResponse> => config.fake ? Promise.resolve(null) : fetchEngagement(config)(parameters),
categories: (
// deno-lint-ignore no-explicit-any
_parent: any,
- ): Promise<CategoriesResponse> =>
- config.fake
- ? Promise.resolve({ data: [] })
- : fetchCategories(config.voltastics),
- engagements: (
+ ): Promise<CategoriesResponse> => config.fake ? Promise.resolve({ data: [] }) : fetchCategories(config.voltastics),
+ filteredEngagements: (
// deno-lint-ignore no-explicit-any
_parent: any,
- parameters: EngagementsParameters,
+ parameters: FilteredEngagementsParameters,
): Promise<EngagementsResponse> =>
- config.fake
- ? Promise.resolve({ totalResults: 0, data: [] })
- : fetchEngagements(config)(parameters),
- cities: (
- // deno-lint-ignore no-explicit-any
- _parent: any,
- ): Promise<CitiesResponse> =>
- config.fake
- ? Promise.resolve({ data: [] })
- : fetchCities(config.voltastics),
+ config.fake ? Promise.resolve({ totalResults: 0, data: [] }) : volunteeringDB.filterEngagements(parameters),
engagementRecommendations: (
// deno-lint-ignore no-explicit-any
_parent: any,
parameters: EngagementRecommendationsParameters,
): Promise<EngagementsResponse> =>
- config.fake
- ? Promise.resolve({ totalResults: 0, data: [] })
- : volunteeringDB.findRecommendations(parameters),
+ config.fake ? Promise.resolve({ totalResults: 0, data: [] }) : volunteeringDB.findRecommendations(parameters),
},
Mutation: {
trackEngagementView: (
@@ -145,15 +130,14 @@ const createResolvers = (
_parent: any,
parameters: TrackEngagementViewParameters,
): Promise<TrackEngagementViewResponse> =>
- config.fake
- ? Promise.resolve({ id: parameters.id })
- : trackEngagementView(config.voltastics)(parameters),
+ config.fake ? Promise.resolve({ id: parameters.id }) : trackEngagementView(config.voltastics)(parameters),
},
});
export const DEFAULT_PORT = 8004;
export const DEFAULT_CACHE_ENABLED = true;
-export const DEFAULT_CACHE_TTL_MS_VOLTASTICS = 60_000;
+export const DEFAULT_CACHE_TTL_MS_VOLTASTICS = 86_400_000; // 24 hours
+export const DEFAULT_CACHE_TTL_MS_DB = 3_600_000; // 1 hour
export const DEFAULT_DB_CONNECTION_ATTEMPTS = 10;
@@ -164,7 +148,8 @@ export interface VoltasticsConfig {
export interface ServerConfig {
port: number; // default: 8004
cacheEnabled: boolean; // default: true
- cacheTtlMsVoltastics: number; // default: 60 seconds
+ cacheTtlMsVoltastics: number; // default: 24 hours
+ cacheTtlMsDB: number; // default: 1 hour
voltastics: VoltasticsConfig;
imageProxyBaseUrl: string;
fake: boolean; // For local development. If set, the API returns dummy data
@@ -181,8 +166,8 @@ export const createGraphQLServer = (config: ServerConfig): GraphQLServer => {
"Query.engagementOpportunities": config.cacheTtlMsVoltastics,
"Query.engagement": config.cacheTtlMsVoltastics,
"Query.categories": config.cacheTtlMsVoltastics,
- "Query.engagements": config.cacheTtlMsVoltastics,
- "Query.cities": config.cacheTtlMsVoltastics,
+ "Query.filteredEngagements": config.cacheTtlMsDB,
+ "Query.engagementRecommendations": config.cacheTtlMsDB,
},
}),
]
@@ -205,10 +190,11 @@ export const createGraphQLServer = (config: ServerConfig): GraphQLServer => {
attempts: config.volunteeringDB.connectionAttempts,
},
});
+ const geoAPIClient = new GeoAPIClient(config.geoAPIEndpointUrl);
const volunteeringDB = new VolunteeringDB(
client,
config.imageProxyBaseUrl,
- resolveGeoLocationViaGeoAPI(config.geoAPIEndpointUrl),
+ geoAPIClient,
);
const resolvers = createResolvers(config, volunteeringDB);
return createYoga({
@@ -227,9 +213,7 @@ export const startServer = (config: ServerConfig): Promise<void> => {
port: config.port,
onListen({ port, hostname }) {
logger.info(
- `Server started at http://${
- hostname === "0.0.0.0" ? "localhost" : hostname
- }:${port}/graphql`,
+ `Server started at http://${hostname === "0.0.0.0" ? "localhost" : hostname}:${port}/graphql`,
);
if (config.fake) {
logger.info(
diff --git a/app/types.ts b/app/types.ts
index 078a6d171d96b2722c485c614a1643cca37c147b..75530ad5b1219217cfe275fd51b79d36ed7a58d5 100644
--- a/app/types.ts
+++ b/app/types.ts
@@ -19,6 +19,10 @@ export type Engagement = {
location?: string;
latitude?: number | null;
longitude?: number | null;
+
+ matchedTopics?: string[];
+ matchedSkills?: string[];
+ matchedGeoLocationDistanceInMeters?: number;
};
export type Category = {
@@ -71,29 +75,20 @@ export type GeolocationGeometry = {
coordinates: number[] | number[][] | number[][][] | number[][][][];
};
-export type Place = {
- lon: number;
+export type GeolocationCoordinates = {
lat: number;
+ lon: number;
};
export type GeolocationGeoJSON = {
- properties: Place;
+ properties: GeolocationCoordinates;
geometry: GeolocationGeometry;
};
-// TODO: Delete the following when app version 1.5.1 is no longer supported.
-export type City = {
- name: string;
-};
-
-export type CitiesResponse = { data: City[] };
-
-export type EngagementsParameters = {
+export type FilteredEngagementsParameters = {
limit: number;
offset: number;
- latitude?: number;
- longitude?: number;
- radiusKm?: number;
- location?: string;
- category?: string;
+ skills: string[];
+ topics: string[];
+ geolocationId?: string;
};
diff --git a/app/voltastics.ts b/app/voltastics.ts
index 39965902bbd0be885d1057619a46966018d75cae..1d32797ecd633e018822b722683aecda3bbbf931 100644
--- a/app/voltastics.ts
+++ b/app/voltastics.ts
@@ -1,25 +1,20 @@
import {
ApiCategoriesResponse,
- ApiCitiesResponse,
ApiDefaults,
ApiEngagementResponse,
ApiRoutes,
ApiSearchEngagement,
ApiSearchEngagementsResponse,
} from "./api_types.ts";
-import { sortCitiesAlphabetically } from "./helpers.ts";
import { logger } from "./logging.ts";
import { ServerConfig, VoltasticsConfig } from "./server.ts";
import {
CategoriesResponse,
Category,
- CitiesResponse,
- City,
Engagement,
EngagementOpportunitiesParameters,
EngagementParameters,
EngagementResponse,
- EngagementsParameters,
EngagementsResponse,
Organizer,
TrackEngagementViewParameters,
@@ -215,8 +210,7 @@ export const trackEngagementView = (voltasticsConfig: VoltasticsConfig) =>
};
export const fetchEngagement =
- (config: ServerConfig) =>
- ({ id }: EngagementParameters): Promise<EngagementResponse> => {
+ (config: ServerConfig) => ({ id }: EngagementParameters): Promise<EngagementResponse> => {
const searchParams = new URLSearchParams();
searchParams.append("id", id);
@@ -278,104 +272,3 @@ export const fetchCategories = (
throw e;
});
};
-
-// TODO: Delete the following when app version 1.5.1 is no longer supported.
-const transformCity = (city: string): City => {
- return {
- name: city,
- };
-};
-
-const transformCitiesResponse = (
- citiesResponse: ApiCitiesResponse,
-): CitiesResponse => {
- return {
- data: citiesResponse.sort(sortCitiesAlphabetically).map(transformCity),
- };
-};
-
-const buildVoltasticsEngagementsSearchParams = ({
- limit = 5,
- offset = 0,
- location = ApiDefaults.CITY,
- category,
- latitude,
- longitude,
- radiusKm,
-}: EngagementsParameters) => {
- const params = new URLSearchParams();
- params.append("limit", limit.toString());
- params.append("offset", offset.toString());
- params.append("city", location);
-
- if (latitude) {
- params.append("lat", latitude.toString());
- }
-
- if (longitude) {
- params.append("lon", longitude.toString());
- }
-
- if (radiusKm) {
- params.append("radius", radiusKm.toString());
- }
-
- if (category) {
- params.append("use", category);
- }
-
- return params;
-};
-
-export const fetchEngagements =
- (config: ServerConfig) =>
- (params: EngagementsParameters): Promise<EngagementsResponse> => {
- const searchParams = buildVoltasticsEngagementsSearchParams(params);
- const start = Date.now();
- logger.info(`fetching engagements from ${config.voltastics.baseUrl}`);
-
- return fetchFromVoltasticsApi(
- config.voltastics,
- ApiRoutes.SEARCH_ENGAGEMENTS,
- searchParams,
- )
- .then((result) => result.json())
- .then(transformEngagementsResponse(config.imageProxyBaseUrl))
- .then((result) => {
- const duration = Date.now() - start;
- logger.debug(`fetching engagements took ${duration} ms`);
- return result;
- })
- .catch((e) => {
- const duration = Date.now() - start;
- logger.error(
- `Error performing request to ${config.voltastics.baseUrl} after ${duration} ms: ${e.message}`,
- { duration },
- );
- throw e;
- });
- };
-
-export const fetchCities = (
- voltasticsConfig: VoltasticsConfig,
-): Promise<CitiesResponse> => {
- const start = Date.now();
- logger.info(`fetching cities from ${voltasticsConfig.baseUrl}`);
-
- return fetchFromVoltasticsApi(voltasticsConfig, ApiRoutes.CITIES)
- .then((result) => result.json())
- .then(transformCitiesResponse)
- .then((result) => {
- const duration = Date.now() - start;
- logger.debug(`fetching cities took ${duration} ms`);
- return result;
- })
- .catch((e) => {
- const duration = Date.now() - start;
- logger.error(
- `Error performing request to ${voltasticsConfig.baseUrl} after ${duration} ms: ${e.message}`,
- { duration },
- );
- throw e;
- });
-};
diff --git a/app/voltastics_test.ts b/app/voltastics_test.ts
index 3150b8f345c613d76d1bcb138772f5c7d17fb7e8..9e53d22d02024833e27fc63e2c3e5452ba8256b5 100644
--- a/app/voltastics_test.ts
+++ b/app/voltastics_test.ts
@@ -11,17 +11,8 @@ import {
} from "./dev_deps.ts";
import { processGqlRequest, stubFetch } from "./common_test.ts";
-import {
- CategoriesResponse,
- Engagement,
- EngagementsResponse,
-} from "./types.ts";
-import {
- createGraphQLServer,
- GraphQLServer,
- ServerConfig,
- VoltasticsConfig,
-} from "./server.ts";
+import { CategoriesResponse, Engagement, EngagementsResponse } from "./types.ts";
+import { createGraphQLServer, GraphQLServer, ServerConfig, VoltasticsConfig } from "./server.ts";
import {
apiCategoriesResponse,
@@ -32,11 +23,7 @@ import {
engagementsResponse,
Munich,
} from "./voltastics_test_data.ts";
-import {
- fetchCategories,
- fetchEngagement,
- fetchEngagementOpportunities,
-} from "./voltastics.ts";
+import { fetchCategories, fetchEngagement, fetchEngagementOpportunities } from "./voltastics.ts";
import { GraphQLError } from "./deps.ts";
const emptyResponse = {
@@ -134,6 +121,7 @@ const serverConfigMock: ServerConfig = {
voltastics: voltasticsConfigMock,
imageProxyBaseUrl: "https://dev-images.holi.social",
cacheTtlMsVoltastics: 0,
+ cacheTtlMsDB: 0,
fake: false,
volunteeringDB: {
hostname: "fakehost",
@@ -152,6 +140,7 @@ const noCacheServerConfig = {
imageProxyBaseUrl: serverConfigMock.imageProxyBaseUrl,
port: 0,
cacheTtlMsVoltastics: 0,
+ cacheTtlMsDB: 0,
fake: false,
volunteeringDB: {
hostname: "fakehost",
diff --git a/app/voltastics_test_data.ts b/app/voltastics_test_data.ts
index 5772426590efb82a572ca1258a1c29b0eb6fca7a..92550289ffd8c9c85d9d6f71f73d40508d25484a 100644
--- a/app/voltastics_test_data.ts
+++ b/app/voltastics_test_data.ts
@@ -1,8 +1,4 @@
-import {
- ApiCategoriesResponse,
- ApiSearchEngagement,
- ApiSearchEngagementsResponse,
-} from "./api_types.ts";
+import { ApiCategoriesResponse, ApiSearchEngagement, ApiSearchEngagementsResponse } from "./api_types.ts";
import { CategoriesResponse, EngagementsResponse } from "./types.ts";
import { transformEngagement } from "./voltastics.ts";
@@ -18,8 +14,7 @@ export const apiSearchEngagement1: ApiSearchEngagement = {
description:
"<p>Weitsprung ist ein gemeinnütziger Reiseanbieter, der sich auf die Bedürfnisse von Reisenden mit Handicap eingestellt hat und seinen Kund:innen während der gesamten Urlaubszeit Hilfen in Form von persönlicher Assistenz bietet. Es reisen Reisebegleiter:innen mit, die die persönlichen Assistenzen geben und dort unterstützen, wo Hilfe gewünscht und gebraucht wird.</p>\n<p>Das ganze Jahr über finden Gruppenreisen mit 4-10 Reisegästen statt. Reiseziele sind Nordsee, Ostsee, Lüneburger Heide, aber auch die Balearen, Kanaren sowie USA, Vietnam und Südafrika. Die Reisedauer liegt zwischen 8 und 21 Tagen.</p>\n<p><strong>Für diese Reisen sucht Weitsprung ehrenamtliche Reisebegleiter:innen. </strong>Die Reisebegleiter:innen geben den Reisenden mit Handicap alle Hilfen vom Koffertragen über Mobilität, Orientierung, Kommmunikation bis hin zur grundpflegerischen Versorgung und vieles mehr.</p>\n<p>Eine Reisegruppe besteht in der Regel aus 4-10 Personen und wird von 2-4 Reisebegleiter:innen unterstützt. Sie werden für diese Aufgabe geschult und fachlich angeleitet. Die ehrenamtlichen Reisebegleiter:innen sind unfall- und haftpflichtversichert. Reisekosten entstehen nicht, die Kosten der Fahrt/Flug, Unterbringung ggf. Halbpension übernimmt der zu begleitende Reisende.</p>\n<p>Der Wohnort der Reisebegleiter und -begleiterinnen kann sich im gesamten Bundesgebiet befinden. Die Vorbereitungstreffen zu den Reisen finden online statt. Ein Online-Einsteigerseminar bereitet Sie auf die Reise vor (2 Tage, jeweils von 10-14 Uhr, an einem Wochenende).</p>\n<p>Gesucht werden reisefreudige Freiwillige, die gern im Team arbeiten und Herausforderungen mögen. Außerdem sollten Sie sich vorstellen können, regelmäßig zu unterstützen - also nicht von vornherein nur eine Reise begleiten wollen. Wie häufig Sie das machen, entscheiden Sie selbst, auch eine Reise pro Jahr ist für uns hilfreich.</p>\n<p>Bitte beachten Sie auch, dass Sie für dieses Engagement gegen Corona geimpft sein müssen. Wenn Sie sich für dieses Engagement interessieren, melden Sie sich gerne in der Freiwilligenagentur.</p>",
source: "Aktion Mensch",
- title:
- "Ehrenamtliche Reisebegleiter:innen für Menschen mit Beeinträchtigungen gesucht",
+ title: "Ehrenamtliche Reisebegleiter:innen für Menschen mit Beeinträchtigungen gesucht",
hashTags: [
"Munich",
"Aktion Mensch",
@@ -34,8 +29,7 @@ export const apiSearchEngagement1: ApiSearchEngagement = {
export const apiSearchEngagement2: ApiSearchEngagement = {
id: 75434,
- image:
- "https://test.com/search/image?id=75434-ehrenamt-im-ambulanten-kinderhospizdienst-hhansestrolche",
+ image: "https://test.com/search/image?id=75434-ehrenamt-im-ambulanten-kinderhospizdienst-hhansestrolche",
latitude: null,
link:
"https://www.aktion-mensch.de/was-du-tun-kannst/ehrenamt/engagement-plattform#/lokale-angebote/f55690cd-e982-4f28-afa6-a9f1b87d8c11?utm_source=voltastics",
@@ -263,8 +257,7 @@ export const Munich = {
"abbreviation_STD": "CET",
"abbreviation_DST": "CEST",
},
- "place_id":
- "51c69eecf83f18274059dd3ad3f89a134840f00101f901dcf3000000000000c002099203084dc3bc6e6368656e",
+ "place_id": "51c69eecf83f18274059dd3ad3f89a134840f00101f901dcf3000000000000c002099203084dc3bc6e6368656e",
},
"geometry": {
"type": "MultiPolygon",
diff --git a/app/volunteering_db.ts b/app/volunteering_db.ts
index 0d0d8af3be520a3d82bee2681768f953036338dc..baa778f73eb1b825612cf3eae5cdb39527867a3e 100644
--- a/app/volunteering_db.ts
+++ b/app/volunteering_db.ts
@@ -1,11 +1,16 @@
-import { Client } from "./deps.ts";
import type { QueryObjectResult } from "./deps.ts";
+import { Client } from "./deps.ts";
+import { GeoAPIClient } from "./geo_api_client.ts";
import {
Engagement,
EngagementRecommendationsParameters,
EngagementsResponse,
+ FilteredEngagementsParameters,
+ GeolocationCoordinates,
+ GeolocationGeometry,
} from "./types.ts";
import { isNonEmptyString, safeJsonParse } from "./utils.ts";
+import { logger } from "./logging.ts";
export type VolunteeringDBConfig = {
hostname: string;
@@ -29,10 +34,13 @@ type VolunteeringDBRow = {
location?: string;
latitude?: number;
longitude?: number;
+ embedding_array: string; // JSON containing `number[]`
cosine_similarity: number;
+ distance_in_meters: number;
+ total_results: number;
};
-const topicsVectorIndices = new Map<string, number>([
+const TOPICS_VECTOR_INDICES = new Map<string, number>([
["agriculture-food", 0],
["alternative-economy", 1],
["biodiversity-animal-welfare", 2],
@@ -63,7 +71,7 @@ const topicsVectorIndices = new Map<string, number>([
["water-ocean", 27],
]);
-const skillsVectorIndices = new Map<string, number>([
+const SKILLS_VECTOR_INDICES = new Map<string, number>([
["adaptability", 0],
["analytical-thinking", 1],
["arts-music-culture", 2],
@@ -116,31 +124,121 @@ const skillsVectorIndices = new Map<string, number>([
["writing-translation", 49],
]);
-const recosQueryVector = (topics: string[], skills: string[]): number[] => {
- const queryVector: number[] = Array(
- topicsVectorIndices.size + skillsVectorIndices.size,
- ).fill(0.0);
+const TOPICS_VECTOR_INDICES_REVERSE = new Map<number, string>(
+ Array.from(TOPICS_VECTOR_INDICES.entries()).map((
+ [key, index],
+ ) => [index, key]),
+);
+
+const SKILLS_VECTOR_INDICES_REVERSE = new Map<number, string>(
+ Array.from(SKILLS_VECTOR_INDICES.entries()).map((
+ [key, index],
+ ) => [index, key]),
+);
+
+const EMBEDDING_DIMENSIONS = TOPICS_VECTOR_INDICES.size +
+ SKILLS_VECTOR_INDICES.size;
+
+/** When annotating search results with matches, embedding values below these thresholds are not considered a match.
+ * Should be aligned with the precomputed binary_embedding.
+ */
+const MATCH_TOPIC_CONFIDENCE_THRESHOLD = 0.8;
+const MATCH_SKILL_CONFIDENCE_THRESHOLD = 0.8;
+const MAX_LIMIT = 100;
+const DEFAULT_LIMIT = 10;
+
+const queryEmbeddingVector = (topics: string[], skills: string[]): number[] => {
+ const queryVector: number[] = Array(EMBEDDING_DIMENSIONS).fill(0.0);
for (const topic of topics) {
- if (topicsVectorIndices.has(topic)) {
- queryVector[topicsVectorIndices.get(topic)!] = 1.0;
+ if (TOPICS_VECTOR_INDICES.has(topic)) {
+ queryVector[TOPICS_VECTOR_INDICES.get(topic)!] = 1.0;
}
}
for (const skill of skills) {
- if (skillsVectorIndices.has(skill)) {
+ if (SKILLS_VECTOR_INDICES.has(skill)) {
queryVector[
- topicsVectorIndices.size + skillsVectorIndices.get(skill)!
+ TOPICS_VECTOR_INDICES.size + SKILLS_VECTOR_INDICES.get(skill)!
] = 1.0;
}
}
return queryVector;
};
+/**
+ * Sorts matches with confidence so that
+ *
+ * * queried for terms come first, ordered by confidence DESC
+ * * not queried for terms come after, ordered by confidence DESC
+ */
+const sortMatches = (
+ queriedTerms: string[],
+ matchedTermsWithConfidence: [string, number][],
+): [string, number][] => {
+ const byConfidenceDesc = (a: [string, number], b: [string, number]) => b[1] - a[1];
+
+ return [
+ ...matchedTermsWithConfidence.filter(([topic, _confidence]) => queriedTerms.includes(topic)).sort(byConfidenceDesc),
+ ...matchedTermsWithConfidence.filter(([topic, _confidence]) => !queriedTerms.includes(topic)).sort(
+ byConfidenceDesc,
+ ),
+ ];
+};
+
+const decodeMatchesFromEmbedding = (
+ embedding: number[],
+ queriedTopics: string[],
+ queriedSkills: string[],
+): { matchedTopics: string[]; matchedSkills: string[] } => {
+ if (embedding.length != EMBEDDING_DIMENSIONS) {
+ logger.warn(
+ `Received embedding of dimension ${embedding.length}, expected ${EMBEDDING_DIMENSIONS}`,
+ );
+ }
+ const skillsOffset = TOPICS_VECTOR_INDICES.size;
+ const matchedTopicsWithConfidence = embedding
+ .map((confidence, index): [string, number] | null => {
+ return index < skillsOffset ? [TOPICS_VECTOR_INDICES_REVERSE.get(index)!, confidence] : null;
+ })
+ .filter((matchWithConfidence) => matchWithConfidence != null)
+ .filter((matchWithConfidence) => matchWithConfidence[1] >= MATCH_TOPIC_CONFIDENCE_THRESHOLD);
+ const matchedSkillsWithConfidence = embedding
+ .map((confidence, index): [string, number] | null => {
+ return index >= skillsOffset ? [SKILLS_VECTOR_INDICES_REVERSE.get(index - skillsOffset)!, confidence] : null;
+ })
+ .filter((matchWithConfidence) => matchWithConfidence != null)
+ .filter((matchWithConfidence) => matchWithConfidence[1] >= MATCH_SKILL_CONFIDENCE_THRESHOLD);
+
+ const withoutConfidence = (a: [string, number]) => a[0];
+ return {
+ matchedTopics: sortMatches(queriedTopics, matchedTopicsWithConfidence).map(
+ withoutConfidence,
+ ),
+ matchedSkills: sortMatches(queriedSkills, matchedSkillsWithConfidence).map(
+ withoutConfidence,
+ ),
+ };
+};
+
export const exportedForTesting = {
- recosQueryVector,
+ queryEmbeddingVector,
+ decodeMatchesFromEmbedding,
+ sortMatches,
+ TOPICS_VECTOR_INDICES,
+ SKILLS_VECTOR_INDICES,
};
-const toEngagement =
- (imageProxyBaseUrl: string) => (row: VolunteeringDBRow): Engagement => ({
+const toEngagement = (
+ imageProxyBaseUrl: string,
+ queriedTopics: string[] = [],
+ queriedSkills: string[] = [],
+) =>
+(row: VolunteeringDBRow): Engagement => {
+ const { matchedTopics, matchedSkills } = decodeMatchesFromEmbedding(
+ JSON.parse(row.embedding_array) as number[],
+ queriedTopics,
+ queriedSkills,
+ );
+ return {
id: "" + row.id,
title: row.title,
description: row.description,
@@ -163,49 +261,52 @@ const toEngagement =
location: row.location,
latitude: row.latitude,
longitude: row.longitude,
- });
-export type GeoLocation = {
- lat: number;
- lon: number;
+ matchedTopics,
+ matchedSkills,
+ matchedGeoLocationDistanceInMeters: row.distance_in_meters,
+ };
};
-export type GeoLocationResolver = (
- geoLocationId: string,
-) => Promise<GeoLocation>;
+const extractTotalResultCount = (
+ queryResult: QueryObjectResult<VolunteeringDBRow>,
+) =>
+ (queryResult.rows.length ? Number(queryResult.rows[0].total_results) : queryResult.rowCount) ??
+ 0;
export class VolunteeringDB {
private client: Client;
private readonly imageProxyBaseUrl: string;
- private resolveGeoLocation: GeoLocationResolver;
+ private geoAPIClient: GeoAPIClient;
constructor(
client: Client,
imageProxyBaseUrl: string,
- resolveGeoLocation: GeoLocationResolver,
+ geoAPIClient: GeoAPIClient,
) {
this.client = client;
this.imageProxyBaseUrl = imageProxyBaseUrl;
- this.resolveGeoLocation = resolveGeoLocation;
+ this.geoAPIClient = geoAPIClient;
}
async findRecommendations(
- { offset, limit, topics, skills, geolocationId }:
- EngagementRecommendationsParameters,
+ { offset, limit, topics, skills, geolocationId }: EngagementRecommendationsParameters,
): Promise<EngagementsResponse> {
const result = await this.queryRecos(
offset < 0 ? 0 : offset,
- limit > 100 || limit < 1 ? 10 : limit,
+ limit > MAX_LIMIT || limit < 1 ? DEFAULT_LIMIT : limit,
topics,
skills,
geolocationId,
).catch((reason) => {
- console.error("Error retrieving recommendations. Reason: ", reason);
+ logger.error("Error retrieving recommendations. Reason: ", reason);
throw reason;
});
- const recos = result.rows.map(toEngagement(this.imageProxyBaseUrl));
+ const recos = result.rows.map(
+ toEngagement(this.imageProxyBaseUrl, topics, skills),
+ );
return Promise.resolve({
- totalResults: recos.length,
+ totalResults: extractTotalResultCount(result),
data: recos,
});
}
@@ -218,21 +319,19 @@ export class VolunteeringDB {
geolocationId?: string,
): Promise<QueryObjectResult<VolunteeringDBRow>> {
const beforeResolve = Date.now();
- const geoLocation = geolocationId
- ? await this.resolveGeoLocation(geolocationId)
- .then((geoLocation) => {
+ const geolocationCoordinates = geolocationId
+ ? await this.geoAPIClient.resolveCoordinates(geolocationId)
+ .then((coordinates) => {
const afterResolve = Date.now();
- console.debug(
- `[${
- (afterResolve - beforeResolve).toString().padStart(4, " ")
- } ms] Successfully resolved ${
- JSON.stringify(geoLocation)
+ logger.debug(
+ `[${(afterResolve - beforeResolve).toString().padStart(4, " ")} ms] Successfully resolved ${
+ JSON.stringify(coordinates)
} for geolocationId=${geolocationId}`,
);
- return geoLocation;
+ return coordinates;
})
.catch((error) => {
- console.warn(error);
+ logger.warn(error);
if (topics.length == 0 && skills.length == 0) {
throw new Error(
"Can't retrieve recommendations: geo location resolution failed and no topics or skills given",
@@ -248,7 +347,7 @@ export class VolunteeringDB {
result: QueryObjectResult<VolunteeringDBRow>,
): QueryObjectResult<VolunteeringDBRow> => {
const afterQuery = Date.now();
- console.debug(
+ logger.debug(
`[${
(afterQuery - beforeQuery).toString().padStart(4, " ")
} ms] Successfully retrieved ${result.rows.length} recommendations from DB`,
@@ -256,16 +355,20 @@ export class VolunteeringDB {
return result;
};
- if (geoLocation && (topics.length > 0 || skills.length > 0)) {
+ if (geolocationCoordinates && (topics.length > 0 || skills.length > 0)) {
return this.queryRecosBasedOnTopicsSkillsAndLocation(
offset,
limit,
topics,
skills,
- geoLocation,
+ geolocationCoordinates,
).then(logQueryDuration);
- } else if (geoLocation) {
- return this.queryRecosBasedOnLocation(offset, limit, geoLocation).then(
+ } else if (geolocationCoordinates) {
+ return this.queryRecosBasedOnLocation(
+ offset,
+ limit,
+ geolocationCoordinates,
+ ).then(
logQueryDuration,
);
} else if (topics.length > 0 || skills.length > 0) {
@@ -288,17 +391,20 @@ export class VolunteeringDB {
topics: string[],
skills: string[],
): Promise<QueryObjectResult<VolunteeringDBRow>> {
- const queryVector = JSON.stringify(recosQueryVector(topics, skills));
+ logger.debug(
+ `queryRecosBasedOnTopicsAndSkills ${JSON.stringify({ offset, limit, topics, skills })}`,
+ );
+ const queryVector = JSON.stringify(queryEmbeddingVector(topics, skills));
return await this.client.queryObject<VolunteeringDBRow>`
WITH vector_matches AS (
SELECT *, 1 - (embedding_array <=> ${queryVector}) AS cosine_similarity
FROM volunteering_voltastics_with_classification
ORDER BY cosine_similarity DESC
- OFFSET ${offset}
- LIMIT ${limit}
)
- SELECT *
- FROM vector_matches;
+ SELECT *, count(*) OVER() AS total_results
+ FROM vector_matches
+ OFFSET ${offset}
+ LIMIT ${limit};
`;
}
@@ -307,13 +413,18 @@ export class VolunteeringDB {
limit: number,
topics: string[],
skills: string[],
- geoLocation: GeoLocation,
+ geolocationCoordinates: GeolocationCoordinates,
): Promise<QueryObjectResult<VolunteeringDBRow>> {
+ logger.debug(
+ `queryRecosBasedOnTopicsSkillsAndLocation ${
+ JSON.stringify({ offset, limit, topics, skills, geolocationCoordinates })
+ }`,
+ );
const rankingWeightCosineSimilarity = 0.7; // Weight for cosine similarity
const rankingWeightDistance = 0.3; // Weight for proximity
const maxDistanceInMeters = 50_000; // in meters (e.g., 50 km)
- const queryVector = JSON.stringify(recosQueryVector(topics, skills));
- const { lat, lon } = geoLocation;
+ const queryVector = JSON.stringify(queryEmbeddingVector(topics, skills));
+ const { lat, lon } = geolocationCoordinates;
// Useful knowledge
// - PostGIS uses lon, lat (NOT lat, lon)
@@ -327,13 +438,14 @@ export class VolunteeringDB {
1 - (embedding_array <=> ${queryVector}) AS cosine_similarity,
ST_Transform(location_gps, 3857) <#> ST_Transform(ST_SetSRID(ST_MakePoint(${lon.toString()}, ${lat.toString()}), 4326), 3857) AS distance_in_meters
FROM volunteering_voltastics_with_classification
+ WHERE longitude IS NOT NULL AND latitude IS NOT NULL
), scored AS (
SELECT *,
(${rankingWeightCosineSimilarity} * cosine_similarity) +
(${rankingWeightDistance} * (1 - LEAST(distance_in_meters, ${maxDistanceInMeters})) / ${maxDistanceInMeters}) AS weighted_score
FROM calculations
)
- SELECT *
+ SELECT *, count(*) OVER() AS total_results
FROM scored
ORDER BY weighted_score DESC
OFFSET ${offset}
@@ -344,11 +456,15 @@ export class VolunteeringDB {
private async queryRecosBasedOnLocation(
offset: number,
limit: number,
- geoLocation: GeoLocation,
+ geolocationCoordinates: GeolocationCoordinates,
): Promise<QueryObjectResult<VolunteeringDBRow>> {
- const { lat, lon } = geoLocation;
+ logger.debug(
+ `queryRecosBasedOnLocation ${JSON.stringify({ offset, limit, geolocationCoordinates })}`,
+ );
+ const { lat, lon } = geolocationCoordinates;
return await this.client.queryObject<VolunteeringDBRow>`
- SELECT *,
+ SELECT *,
+ count(*) OVER() AS total_results,
ST_Transform(location_gps, 3857) <#> ST_Transform(ST_SetSRID(ST_MakePoint(${lon.toString()}, ${lat.toString()}), 4326), 3857) AS distance_in_meters
FROM volunteering_voltastics_with_classification
ORDER BY distance_in_meters ASC
@@ -356,4 +472,93 @@ export class VolunteeringDB {
LIMIT ${limit};
`;
}
+
+ async filterEngagements(
+ { offset, limit, topics, skills, geolocationId }: FilteredEngagementsParameters,
+ ): Promise<EngagementsResponse> {
+ const geometry: GeolocationGeometry | undefined = geolocationId
+ ? await this.geoAPIClient.resolveGeometry(geolocationId).catch(
+ (error) => {
+ logger.warn(error);
+ throw new Error(
+ "Can't filter engagements: Geolocation resolution failed",
+ );
+ },
+ )
+ : undefined;
+
+ if (!geometry && !(topics.length + skills.length)) {
+ throw new Error(
+ "Can't filter engagements: At least one topic, skill or location is required",
+ );
+ }
+
+ return this.queryFilteredEngagements(
+ topics,
+ skills,
+ offset < 0 ? 0 : offset,
+ limit > MAX_LIMIT || limit < 1 ? DEFAULT_LIMIT : limit,
+ geometry,
+ );
+ }
+
+ private async queryFilteredEngagements(
+ topics: string[],
+ skills: string[],
+ offset: number,
+ limit: number,
+ geometry?: GeolocationGeometry,
+ ): Promise<EngagementsResponse> {
+ const queryVector = queryEmbeddingVector(topics, skills);
+ const queryVectorString = JSON.stringify(queryVector);
+ const nonEmptyQueryVector = topics.length + skills.length > 0;
+ const binaryQueryVector = queryVector.map((value) => value > 0 ? "1" : "0")
+ .join("");
+ const emptyBinaryQueryVector = "0".repeat(EMBEDDING_DIMENSIONS);
+ const beforeQuery = Date.now();
+
+ // Filtering engagements:
+ // - Precomputed `binary_embedding` is a bit string marking matching topics and skills
+ // - Unset bits in `binary_embedding` for terms that are not part of the query using bitwise AND => If at least one set bit remains, it matches the query (checked by comparing to empty bit string, as it's too long for integer conversion)
+ // - Filter by geolocation by checking if coordinates are inside the given geolocation's GeoJSON geometry
+ // - Sort results using cosine_similarity
+ const result = await this.client.queryObject<VolunteeringDBRow>`
+ WITH vector_matches AS (
+ SELECT
+ *,
+ 1 - (embedding_array <=> ${queryVectorString}) AS cosine_similarity,
+ CASE
+ WHEN ${nonEmptyQueryVector}
+ THEN (binary_embedding & ${binaryQueryVector}) > ${emptyBinaryQueryVector}
+ ELSE true
+ END AS matches_query,
+ CASE
+ WHEN ${!!geometry}
+ THEN ST_Contains(ST_GeomFromGeoJSON(${geometry}), location_gps)
+ ELSE true
+ END AS matches_location
+ FROM volunteering_voltastics_with_classification
+ ORDER BY cosine_similarity DESC
+ )
+ SELECT *, count(*) OVER() AS total_results
+ FROM vector_matches
+ WHERE matches_query
+ AND matches_location
+ OFFSET ${offset}
+ LIMIT ${limit};
+ `;
+
+ const afterQuery = Date.now();
+ const totalResults = extractTotalResultCount(result);
+ logger.debug(
+ `[${
+ (afterQuery - beforeQuery).toString().padStart(4, " ")
+ } ms] Successfully fetched ${result.rows.length}/${totalResults} filtered engagements from DB`,
+ );
+
+ return {
+ totalResults,
+ data: result.rows.map(toEngagement(this.imageProxyBaseUrl)),
+ };
+ }
}
diff --git a/app/volunteering_db_test.ts b/app/volunteering_db_test.ts
index f26e4399ce7c54379dcc194a827c332f46d0fa51..2a464b646f71fe0f5be654332bd00f65569dfbe7 100644
--- a/app/volunteering_db_test.ts
+++ b/app/volunteering_db_test.ts
@@ -1,4 +1,3 @@
-import type { Spy } from "./dev_deps.ts";
import {
assertEquals,
assertRejects,
@@ -6,138 +5,371 @@ import {
assertSpyCalls,
describe,
it,
- spy,
+ returnsNext,
+ Stub,
+ stub,
} from "./dev_deps.ts";
-import {
- exportedForTesting,
- GeoLocationResolver,
- VolunteeringDB,
-} from "./volunteering_db.ts";
+import { exportedForTesting, VolunteeringDB } from "./volunteering_db.ts";
import { Client } from "./deps.ts";
+import { GeoAPIClient } from "./geo_api_client.ts";
+
+// taken from https://stackoverflow.com/a/2450976 ("Fisher-Yates Shuffle")
+function shuffle<A>(array: A[]): A[] {
+ let currentIndex = array.length;
+ // While there remain elements to shuffle...
+ while (currentIndex != 0) {
+ // Pick a remaining element...
+ const randomIndex = Math.floor(Math.random() * currentIndex);
+ currentIndex--;
-type ResolveLocationSpy = Spy<
- unknown,
- [_geoLocationId: string],
- Promise<{ lat: number; lon: number }>
->;
+ // And swap it with the current element.
+ [array[currentIndex], array[randomIndex]] = [
+ array[randomIndex],
+ array[currentIndex],
+ ];
+ }
+ return array;
+}
-const succeedingGeoLocationResolver: GeoLocationResolver = (
- _geoLocationId: string,
-) => Promise.resolve({ lat: 123, lon: 321 });
-const failingGeoLocationResolver: GeoLocationResolver = (
- _geoLocationId: string,
-) => Promise.reject("boom!");
+const testCoordinates = { lat: 123, lon: 321 };
+const testGeometry = {
+ type: "Polygon",
+ "coordinates": [[[3.9265487, 52.576948399], [13.9304862, 52.574110199]]],
+};
const withMockedDependencies = (
- geoLocationResolver: GeoLocationResolver,
test: (
vDB: VolunteeringDB,
- resolveLocationSpy: ResolveLocationSpy,
+ geoApiClient: GeoAPIClient,
) => unknown | Promise<unknown>,
) => {
- const geoLocationResolverSpy = spy(geoLocationResolver);
const mockClient = {
queryObject: () => Promise.resolve({ rows: [] }),
} as unknown as Client;
+ const mockedGeoApiClient = {} as unknown as GeoAPIClient;
+ stub(
+ mockedGeoApiClient,
+ "resolveCoordinates",
+ returnsNext([Promise.resolve(testCoordinates)]),
+ );
+ stub(
+ mockedGeoApiClient,
+ "resolveGeometry",
+ returnsNext([Promise.resolve(testGeometry)]),
+ );
const volunteeringDB = new VolunteeringDB(
mockClient,
"mock-image-proxy-base-url",
- geoLocationResolverSpy,
+ mockedGeoApiClient,
);
- test(volunteeringDB, geoLocationResolverSpy);
+ test(volunteeringDB, mockedGeoApiClient);
};
describe("VolunteeringDB", () => {
- it("correctly constructs a query vector", () => {
- const expectedLength = 28 /* #topics */ + 50; /* #skills */
- // deno-fmt-ignore
- const expectedVector: number[] = [
- 1,0,0,0,0,0,0,0,0,1,
- 0,0,0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,1,
- 1,0,0,0,0,0,0,0,0,1,
- 0,0,0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0,0,0,
- 0,0,0,0,0,0,0,0,0,1,
- ]
- // deno-fmt-ignore
- const actualVector = exportedForTesting.recosQueryVector([
- "agriculture-food", // first item (index 0)
- "disabilities-inclusion", // tenth item (index 9)
- "water-ocean", // last item (index 27)
- ], [
- "adaptability", // first item (index 0)
- "conflict-management", // tenth item (index 9)
- "writing-translation", // last item (index 49)
- ]);
- assertEquals(actualVector.length, expectedLength);
- assertEquals(actualVector, expectedVector);
+ describe("queryEmbeddingVector", () => {
+ it("correctly constructs a query embedding vector", () => {
+ const expectedLength = 28 /* #topics */ + 50; /* #skills */
+ // deno-fmt-ignore
+ const expectedVector: number[] = [
+ 1,0,0,0,0,0,0,0,0,1,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,1,
+ 1,0,0,0,0,0,0,0,0,1,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,1,
+ ]
+ // deno-fmt-ignore
+ const actualVector = exportedForTesting.queryEmbeddingVector([
+ "agriculture-food", // first item (index 0)
+ "disabilities-inclusion", // tenth item (index 9)
+ "water-ocean", // last item (index 27)
+ ], [
+ "adaptability", // first item (index 0)
+ "conflict-management", // tenth item (index 9)
+ "writing-translation", // last item (index 49)
+ ]);
+ assertEquals(actualVector.length, expectedLength);
+ assertEquals(actualVector, expectedVector);
+ });
});
- it("resolves latitude/longitude using the GeoAPI client when geoLocationId is given", () => {
- withMockedDependencies(
- succeedingGeoLocationResolver,
- (volunteeringDB, resolveLocationSpy) => {
- volunteeringDB.findRecommendations({
- limit: 10,
- offset: 0,
- topics: [],
- skills: [],
- geolocationId: "mock-geolocation-id",
- });
- assertSpyCall(resolveLocationSpy, 0, {
- args: ["mock-geolocation-id"],
- });
- },
- );
+ describe("decodeMatchesFromEmbedding", () => {
+ it("correctly reverse-decodes topics and skills matches from embeddings", () => {
+ // deno-fmt-ignore
+ const embedding: number[] = [
+ 0.9,0,0,0,0,0,0,0,0,0.8,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0.85,
+ 0.85,0,0,-0.5,0,0,0,0,0,0.99,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,1.0,
+ ]
+ // ordered by confidence DESC with queried for terms first
+ const expectedMatchedTopics = [
+ "agriculture-food", // first item (index 0)
+ "water-ocean", // last item (index 27)
+ "disabilities-inclusion", // tenth item (index 9)
+ ];
+ // ordered by confidence DESC with queried for terms first
+ const expectedMatchedSkills = [
+ "writing-translation", // last item (index 49)
+ "conflict-management", // tenth item (index 9)
+ "adaptability", // first item (index 0)
+ ];
+ const allTopics = Array.from(
+ exportedForTesting.TOPICS_VECTOR_INDICES.keys(),
+ );
+ const allSkills = Array.from(
+ exportedForTesting.SKILLS_VECTOR_INDICES.keys(),
+ );
+ const matches = exportedForTesting.decodeMatchesFromEmbedding(
+ embedding,
+ allTopics,
+ allSkills,
+ );
+ assertEquals(matches.matchedTopics, expectedMatchedTopics);
+ assertEquals(matches.matchedSkills, expectedMatchedSkills);
+ });
+ it("filters out topics and skills matches with confidence below MATCH_CONFIDENCE_THRESHOLD", () => {
+ // deno-fmt-ignore
+ const embedding: number[] = [
+ 0.9,0,0,0,0,0,0,0,0,0.3,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0.85,
+ 0.3,0,0,-0.5,0,0,0,0,0,0.99,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,1.0,
+ ]
+ // ordered by confidence DESC with queried for terms first
+ const expectedMatchedTopics = [
+ "agriculture-food", // first item (index 0)
+ "water-ocean", // last item (index 27)
+ ];
+ // ordered by confidence DESC with queried for terms first
+ const expectedMatchedSkills = [
+ "writing-translation", // last item (index 49)
+ "conflict-management", // tenth item (index 9)
+ ];
+ const allTopics = Array.from(
+ exportedForTesting.TOPICS_VECTOR_INDICES.keys(),
+ );
+ const allSkills = Array.from(
+ exportedForTesting.SKILLS_VECTOR_INDICES.keys(),
+ );
+ const matches = exportedForTesting.decodeMatchesFromEmbedding(
+ embedding,
+ allTopics,
+ allSkills,
+ );
+ assertEquals(matches.matchedTopics, expectedMatchedTopics);
+ assertEquals(matches.matchedSkills, expectedMatchedSkills);
+ });
+ it("ranks topics and skills matches queried for higher", () => {
+ // deno-fmt-ignore
+ const embedding: number[] = [
+ 0.85,0,0,0,0,0,0,0,0,0.9,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0.8,0.9,
+ 0.85,0,0,-0.5,0,0,0,0,0,0.99,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,0,
+ 0,0,0,0,0,0,0,0,0,1.0,
+ ]
+ // ordered by confidence DESC with queried for terms first
+ const expectedMatchedTopics = [
+ "disabilities-inclusion", // tenth item (index 9)
+ "agriculture-food", // first item (index 0)
+ "water-ocean", // last item (
+ "urban-rural-development",
+ ];
+ // ordered by confidence DESC with queried for terms first
+ const expectedMatchedSkills = [
+ "writing-translation", // last item (index 49)
+ "conflict-management", // tenth item (index 9)
+ "adaptability", // first item (index 0)
+ ];
+ const queriedTopics = ["agriculture-food", "disabilities-inclusion"];
+ const queriedSkills = ["writing-translation"];
+ const matches = exportedForTesting.decodeMatchesFromEmbedding(
+ embedding,
+ queriedTopics,
+ queriedSkills,
+ );
+ assertEquals(matches.matchedTopics, expectedMatchedTopics);
+ assertEquals(matches.matchedSkills, expectedMatchedSkills);
+ });
});
- it("does not resolve latitude/longitude when geoLocationId is not given", () => {
- withMockedDependencies(
- succeedingGeoLocationResolver,
- (volunteeringDB, resolveLocationSpy) => {
- volunteeringDB.findRecommendations({
- limit: 10,
- offset: 0,
- topics: ["agriculture-food"],
- skills: [],
- });
- assertSpyCalls(resolveLocationSpy, 0);
- },
- );
+
+ describe("sortMatches", () => {
+ it("sortMatches ranks topics and skills matches by confidence descending and queried for terms higher", () => {
+ const sorted = exportedForTesting.sortMatches(
+ ["a", "c"],
+ shuffle([
+ ["a", 0.1],
+ ["b", 0.2],
+ ["c", 0.3],
+ ["d", 0.4],
+ ["e", 0.5],
+ ]),
+ );
+ assertEquals(sorted, [["c", 0.3], ["a", 0.1], ["e", 0.5], ["d", 0.4], [
+ "b",
+ 0.2,
+ ]]);
+ });
});
- it("falls back to recommend based only on topics and skills if given and geo location resolution fails", () => {
- withMockedDependencies(
- failingGeoLocationResolver,
- async (volunteeringDB, _resolveLocationSpy) => {
- const result = await volunteeringDB.findRecommendations({
- limit: 10,
- offset: 0,
- topics: ["agriculture-food"],
- skills: [],
- geolocationId: "mock-geolocation-id",
- });
- assertEquals(result.totalResults, 0);
- assertEquals(result.data, []);
- },
- );
+
+ describe("findRecommendations", () => {
+ it("resolves latitude/longitude using the GeoAPI client when geolocationId is given", () => {
+ withMockedDependencies(
+ (volunteeringDB, geoAPIClient) => {
+ volunteeringDB.findRecommendations({
+ limit: 10,
+ offset: 0,
+ topics: [],
+ skills: [],
+ geolocationId: "mock-geolocation-id",
+ });
+ assertSpyCall(geoAPIClient.resolveCoordinates as Stub, 0, {
+ args: ["mock-geolocation-id"],
+ });
+ },
+ );
+ });
+ it("does not resolve latitude/longitude when geolocationId is not given", () => {
+ withMockedDependencies(
+ (volunteeringDB, geoAPIClient) => {
+ volunteeringDB.findRecommendations({
+ limit: 10,
+ offset: 0,
+ topics: ["agriculture-food"],
+ skills: [],
+ });
+ assertSpyCalls(geoAPIClient.resolveCoordinates as Stub, 0);
+ },
+ );
+ });
+ it("falls back to recommend based only on topics and skills if given and geo location resolution fails", () => {
+ withMockedDependencies(
+ async (volunteeringDB, geoAPIClient) => {
+ stub(
+ geoAPIClient,
+ "resolveCoordinates",
+ returnsNext([Promise.reject("boom!")]),
+ );
+
+ const result = await volunteeringDB.findRecommendations({
+ limit: 10,
+ offset: 0,
+ topics: ["agriculture-food"],
+ skills: [],
+ geolocationId: "mock-geolocation-id",
+ });
+ assertEquals(result.totalResults, 0);
+ assertEquals(result.data, []);
+ },
+ );
+ });
+ it("fails with error message if no topics and skills are given and geo location resolution fails", () => {
+ withMockedDependencies(
+ async (volunteeringDB, geoAPIClient) => {
+ stub(
+ geoAPIClient,
+ "resolveCoordinates",
+ returnsNext([Promise.reject("boom!")]),
+ );
+ await assertRejects(
+ () =>
+ volunteeringDB.findRecommendations({
+ limit: 10,
+ offset: 0,
+ topics: [],
+ skills: [],
+ geolocationId: "mock-geolocation-id",
+ }),
+ Error,
+ "geo location resolution failed and no topics or skills given",
+ );
+ },
+ );
+ });
});
- it("fails with error message if no topics and skills are given and geo location resolution fails", () => {
- withMockedDependencies(
- failingGeoLocationResolver,
- async (volunteeringDB, _resolveLocationSpy) => {
- await assertRejects(
- () =>
- volunteeringDB.findRecommendations({
- limit: 10,
- offset: 0,
- topics: [],
- skills: [],
- geolocationId: "mock-geolocation-id",
- }),
- Error,
- "geo location resolution failed and no topics or skills given",
- );
- },
- );
+
+ describe("filterEngagements", () => {
+ it("resolves geolocation geometry using the GeoAPI client when geolocationId is given", () => {
+ withMockedDependencies(
+ (volunteeringDB, geoAPIClient) => {
+ volunteeringDB.filterEngagements({
+ limit: 10,
+ offset: 0,
+ topics: [],
+ skills: [],
+ geolocationId: "mock-geolocation-id",
+ });
+ assertSpyCall(geoAPIClient.resolveGeometry as Stub, 0, {
+ args: ["mock-geolocation-id"],
+ });
+ },
+ );
+ });
+ it("does not resolve geolocation geometry when geolocationId is not given", () => {
+ withMockedDependencies(
+ (volunteeringDB, geoAPIClient) => {
+ volunteeringDB.filterEngagements({
+ limit: 10,
+ offset: 0,
+ topics: ["agriculture-food"],
+ skills: [],
+ });
+ assertSpyCalls(geoAPIClient.resolveGeometry as Stub, 0);
+ },
+ );
+ });
+ it("fails with error message if geolocation resolution fails", () => {
+ withMockedDependencies(
+ async (volunteeringDB, geoAPIClient) => {
+ stub(
+ geoAPIClient,
+ "resolveGeometry",
+ returnsNext([Promise.reject("boom!")]),
+ );
+ await assertRejects(
+ () =>
+ volunteeringDB.filterEngagements({
+ limit: 10,
+ offset: 0,
+ topics: ["agriculture-food"],
+ skills: [],
+ geolocationId: "mock-geolocation-id",
+ }),
+ Error,
+ "Geolocation resolution failed",
+ );
+ },
+ );
+ });
+ it("fails with error message if no topics, skills and location are given", () => {
+ withMockedDependencies(
+ async (volunteeringDB, _geoAPIClient) => {
+ await assertRejects(
+ () =>
+ volunteeringDB.filterEngagements({
+ limit: 10,
+ offset: 0,
+ topics: [],
+ skills: [],
+ }),
+ Error,
+ "At least one topic, skill or location is required",
+ );
+ },
+ );
+ });
});
});
diff --git a/deno.json b/deno.json
index 184e3eda5b4a4ee8464ba50b1fc65aa248b3c1f0..31f7928cca9237bcc5a6a611695c53e83b627d92 100644
--- a/deno.json
+++ b/deno.json
@@ -19,6 +19,7 @@
}
},
"fmt": {
+ "lineWidth": 120,
"exclude": [
"*.md"
]
diff --git a/terraform/environments/deployment.tf b/terraform/environments/deployment.tf
index 86a05f82e347259aca5ac3fe5b7f730be48f197a..0b4f8cf0873f106697768043e0c20ff5ff172810 100644
--- a/terraform/environments/deployment.tf
+++ b/terraform/environments/deployment.tf
@@ -63,8 +63,12 @@ resource "google_cloud_run_service" "volunteering_api" {
value = "true"
}
env {
- name = "CACHE_TTL_MS"
- value = local.environment == "production" ? "86400000" : "86400000"
+ name = "CACHE_TTL_MS_VOLTASTICS"
+ value = local.environment == "production" ? "86400000" : "86400000" # 24 hours
+ }
+ env {
+ name = "CACHE_TTL_MS_DB"
+ value = local.environment == "production" ? "3600000" : "3600000" # 1 hour
}
env {
name = "VOLUNTEERING_VOLTASTICS_API_URL"
@@ -100,8 +104,8 @@ resource "google_cloud_run_service" "volunteering_api" {
name = "DB_USERNAME"
value_from {
secret_key_ref {
- key = "latest"
- name = local.environment_name == "production" ? "VOLUNTEERING_DB_USERNAME_PRODUCTION" : "VOLUNTEERING_DB_USERNAME_DEVELOPMENT"
+ key = "latest"
+ name = local.environment_name == "production" ? "VOLUNTEERING_DB_USERNAME_PRODUCTION" : "VOLUNTEERING_DB_USERNAME_DEVELOPMENT"
}
}
}
@@ -109,13 +113,13 @@ resource "google_cloud_run_service" "volunteering_api" {
name = "DB_PASSWORD"
value_from {
secret_key_ref {
- key = "latest"
- name = local.environment_name == "production" ? "VOLUNTEERING_DB_PASSWORD_PRODUCTION" : "VOLUNTEERING_DB_PASSWORD_DEVELOPMENT"
+ key = "latest"
+ name = local.environment_name == "production" ? "VOLUNTEERING_DB_PASSWORD_PRODUCTION" : "VOLUNTEERING_DB_PASSWORD_DEVELOPMENT"
}
}
}
env {
- name = "GEO_API_ENDPOINT_URL"
+ name = "GEO_API_ENDPOINT_URL"
value = data.terraform_remote_state.holi_geo_api_environments_state.outputs.api_endpoint_url
}
resources {
@@ -142,6 +146,11 @@ resource "google_cloud_run_service" "volunteering_api" {
# possible values: all-traffic/private-ranges-only(default) https://cloud.google.com/sdk/gcloud/reference/run/services/update#--vpc-egress
"run.googleapis.com/vpc-access-egress" = "all-traffic"
}
+ # labels set on the revision level
+ labels = {
+ "environment_type" = local.environment_type
+ "holi_service" = "app-volunteering"
+ }
}
}
@@ -151,6 +160,11 @@ resource "google_cloud_run_service" "volunteering_api" {
# possible values: all/internal/internal-and-cloud-load-balancing https://cloud.google.com/sdk/gcloud/reference/run/services/update#--ingress
"run.googleapis.com/ingress" = "internal"
}
+ # labels set on the service level
+ labels = {
+ "environment_type" = local.environment_type
+ "holi_service" = "app-volunteering"
+ }
}
traffic {
diff --git a/terraform/environments/vars.tf b/terraform/environments/vars.tf
index e86e9ca7e59147a5c7f4aa0b088ab45514565510..79a7ba570bead16ad2ed6c502640876e89cc90d9 100644
--- a/terraform/environments/vars.tf
+++ b/terraform/environments/vars.tf
@@ -2,6 +2,7 @@ locals {
# predefined environment names are staging, production, and <branchname>
environment = terraform.workspace
environment_name = trim(substr(local.environment, 0, 22), ".-") # limits the length of the name that it fits constraints google imposes for some resources, e.g. project ids
+ environment_type = contains(["staging", "production"], local.environment) ? local.environment : "review"
# the dns name can be at most 64 chars in total
dns_name = "${trimsuffix(substr(local.environment, 0, 34), ".-")}.${data.terraform_remote_state.holi_volunteering_api_common_state.outputs.dns_volunteering_api_domain}"